This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 219d4f224e1d97d2bd7cd5dec5ec5968e89facef
Merge: f9dabfc250 ef213b520b
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Fri May 24 12:45:55 2024 -0400
Merge branch 'main' into elasticity
.../apache/accumulo/core/metadata/schema/Ample.java | 4 ++--
.../core/metadata/schema/TabletMetadata.java | 20 +++++++++-----------
.../core/metadata/schema/TabletMetadataBuilder.java | 5 +++--
.../core/metadata/schema/TabletMutatorBase.java | 11 +++++++----
.../core/metadata/schema/TabletMetadataTest.java | 11 +++++++----
.../accumulo/server/util/MetadataTableUtil.java | 8 ++++----
.../accumulo/server/SetEncodingIteratorTest.java | 13 +++++++++----
.../server/manager/state/TabletManagementTest.java | 15 ++++++++++++---
.../manager/tableOps/merge/DeleteTablets.java | 6 +++---
.../org/apache/accumulo/test/functional/SplitIT.java | 7 +++++--
.../org/apache/accumulo/test/metrics/MetricsIT.java | 4 +++-
11 files changed, 64 insertions(+), 40 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 3c00d70f33,931e415774..d3875ca826
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@@ -20,22 -20,15 +20,22 @@@ package org.apache.accumulo.core.metada
import java.util.Collection;
import java.util.Iterator;
+import java.util.Map;
- import java.util.Set;
import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
import java.util.stream.Stream;
+import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.core.client.admin.TabletAvailability;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
++import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.gc.GcCandidate;
import org.apache.accumulo.core.gc.ReferenceFile;
-import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.ScanServerRefTabletFile;
@@@ -343,93 -265,50 +343,93 @@@ public interface Ample
/**
* Interface for changing a tablets persistent data.
*/
- interface TabletMutator {
- TabletMutator putPrevEndRow(Text per);
+ interface TabletUpdates<T> {
+ T putPrevEndRow(Text per);
+
+ T putFile(ReferencedTabletFile path, DataFileValue dfv);
+
+ T putFile(StoredTabletFile path, DataFileValue dfv);
+
+ T deleteFile(StoredTabletFile path);
+
+ T putScan(StoredTabletFile path);
- TabletMutator putFile(ReferencedTabletFile path, DataFileValue dfv);
+ T deleteScan(StoredTabletFile path);
- TabletMutator putFile(StoredTabletFile path, DataFileValue dfv);
+ T putFlushId(long flushId);
- TabletMutator deleteFile(StoredTabletFile path);
+ T putFlushNonce(long flushNonce);
- TabletMutator putScan(StoredTabletFile path);
+ T putLocation(Location location);
- TabletMutator deleteScan(StoredTabletFile path);
+ T deleteLocation(Location location);
- TabletMutator putCompactionId(long compactionId);
+ T putDirName(String dirName);
- TabletMutator putFlushId(long flushId);
+ T putWal(LogEntry logEntry);
- TabletMutator putLocation(Location location);
+ T deleteWal(LogEntry logEntry);
- TabletMutator deleteLocation(Location location);
+ T putTime(MetadataTime time);
- TabletMutator putZooLock(ServiceLock zooLock);
+ T putBulkFile(ReferencedTabletFile bulkref, FateId fateId);
- TabletMutator putDirName(String dirName);
+ T deleteBulkFile(StoredTabletFile bulkref);
- TabletMutator putWal(LogEntry logEntry);
+ T putSuspension(TServerInstance tserver, SteadyTime suspensionTime);
- TabletMutator deleteWal(LogEntry wal);
+ T deleteSuspension();
- TabletMutator putTime(MetadataTime time);
+ T putExternalCompaction(ExternalCompactionId ecid, CompactionMetadata
ecMeta);
- TabletMutator putBulkFile(ReferencedTabletFile bulkref, long tid);
+ T deleteExternalCompaction(ExternalCompactionId ecid);
- TabletMutator deleteBulkFile(StoredTabletFile bulkref);
+ T putCompacted(FateId fateId);
- TabletMutator putSuspension(TServerInstance tserver, SteadyTime
suspensionTime);
+ T deleteCompacted(FateId fateId);
- TabletMutator deleteSuspension();
+ T putTabletAvailability(TabletAvailability tabletAvailability);
- TabletMutator putExternalCompaction(ExternalCompactionId ecid,
- ExternalCompactionMetadata ecMeta);
+ T setHostingRequested();
- TabletMutator deleteExternalCompaction(ExternalCompactionId ecid);
+ T deleteHostingRequested();
+
+ T putOperation(TabletOperationId opId);
+
+ T deleteOperation();
+
+ T putSelectedFiles(SelectedFiles selectedFiles);
+
+ T deleteSelectedFiles();
+ /**
+ * Deletes all the columns in the keys.
+ *
+ * @throws IllegalArgumentException if rows in keys do not match tablet
row or column visibility
+ * is not empty
+ */
- T deleteAll(Set<Key> keys);
++ T deleteAll(Collection<Map.Entry<Key,Value>> entries);
+
+ T setMerged();
+
+ T deleteMerged();
+
+ T putUserCompactionRequested(FateId fateId);
+
+ T deleteUserCompactionRequested(FateId fateId);
+
+ T setUnSplittable(UnSplittableMetadata unSplittableMeta);
+
+ T deleteUnSplittable();
+
+ /**
+ * By default the server lock is automatically added to mutations unless
this method is set to
+ * false.
+ */
+ T automaticallyPutServerLock(boolean b);
+ }
+
+ interface TabletMutator extends TabletUpdates<TabletMutator> {
/**
* This method persist (or queues for persisting) previous put and
deletes against this object.
* Unless this method is called, previous calls will never be persisted.
The purpose of this
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index ef1fd5ea9c,33d307ce59..59a2c1d760
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -39,12 -37,9 +39,11 @@@ import java.util.Objects
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
- import java.util.SortedMap;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.clientImpl.TabletAvailabilityUtil;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.TableId;
@@@ -87,8 -77,7 +86,7 @@@ import com.google.common.base.Supplier
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedMap.Builder;
+import com.google.common.collect.ImmutableSet;
- import com.google.common.collect.ImmutableSortedMap;
import com.google.common.net.HostAndPort;
public class TabletMetadata {
@@@ -111,20 -100,13 +109,20 @@@
private final String dirName;
private final MetadataTime time;
private final String cloned;
- private final SortedMap<Key,Value> keyValues;
+ private final List<Entry<Key,Value>> keyValues;
private final OptionalLong flush;
+ private final OptionalLong flushNonce;
private final List<LogEntry> logs;
- private final OptionalLong compact;
- private final Double splitRatio;
- private final Map<ExternalCompactionId,ExternalCompactionMetadata>
extCompactions;
+ private final Map<ExternalCompactionId,CompactionMetadata> extCompactions;
private final boolean merged;
+ private final TabletAvailability availability;
+ private final boolean onDemandHostingRequested;
+ private final TabletOperationId operationId;
+ private final boolean futureAndCurrentLocationSet;
+ private final Set<FateId> compacted;
+ private final Set<FateId> userCompactionsRequested;
+ private final UnSplittableMetadata unSplittableMetadata;
+ private final Supplier<Long> fileSize;
private TabletMetadata(Builder tmBuilder) {
this.tableId = tmBuilder.tableId;
@@@ -142,29 -125,14 +140,29 @@@
this.dirName = tmBuilder.dirName;
this.time = tmBuilder.time;
this.cloned = tmBuilder.cloned;
- this.keyValues =
Optional.ofNullable(tmBuilder.keyValues).map(ImmutableSortedMap.Builder::build)
- .orElse(null);
+ this.keyValues =
+
Optional.ofNullable(tmBuilder.keyValues).map(ImmutableList.Builder::build).orElse(null);
this.flush = tmBuilder.flush;
+ this.flushNonce = tmBuilder.flushNonce;
this.logs = Objects.requireNonNull(tmBuilder.logs.build());
- this.compact = Objects.requireNonNull(tmBuilder.compact);
- this.splitRatio = tmBuilder.splitRatio;
this.extCompactions =
Objects.requireNonNull(tmBuilder.extCompactions.build());
this.merged = tmBuilder.merged;
+ this.availability = Objects.requireNonNull(tmBuilder.availability);
+ this.onDemandHostingRequested = tmBuilder.onDemandHostingRequested;
+ this.operationId = tmBuilder.operationId;
+ this.futureAndCurrentLocationSet = tmBuilder.futureAndCurrentLocationSet;
+ this.compacted = tmBuilder.compacted.build();
+ this.userCompactionsRequested =
tmBuilder.userCompactionsRequested.build();
+ this.unSplittableMetadata = tmBuilder.unSplittableMetadata;
+ this.fileSize = Suppliers.memoize(() -> {
+ // This code was using a java stream. While profiling SplitMillionIT,
the stream was showing
+ // up as hot when scanning 1 million tablets. Converted to a for loop
to improve performance.
+ long sum = 0;
+ for (var dfv : files.values()) {
+ sum += dfv.getSize();
+ }
+ return sum;
+ });
this.extent =
Suppliers.memoize(() -> new KeyExtent(getTableId(), getEndRow(),
getPrevEndRow()));
}
@@@ -415,54 -380,34 +413,54 @@@
return merged;
}
- public List<Entry<Key,Value>> getKeyValues() {
- Preconditions.checkState(keyValues != null, "Requested key values when it
was not saved");
- return keyValues;
+ public Set<FateId> getUserCompactionsRequested() {
+ ensureFetched(ColumnType.USER_COMPACTION_REQUESTED);
+ return userCompactionsRequested;
}
- public TabletState getTabletState(Set<TServerInstance> liveTServers) {
- ensureFetched(ColumnType.LOCATION);
- ensureFetched(ColumnType.LAST);
- ensureFetched(ColumnType.SUSPEND);
- try {
- Location current = null;
- Location future = null;
- if (hasCurrent()) {
- current = location;
- } else {
- future = location;
- }
- // only care about the state so don't need walogs and chopped params
- // Use getExtent() when passing the extent as the private reference may
not have been
- // initialized yet. This will also ensure PREV_ROW was fetched
- var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null);
- return tls.getState(liveTServers);
- } catch (TabletLocationState.BadLocationStateException blse) {
- throw new IllegalArgumentException("Error creating
TabletLocationState", blse);
+ public TabletAvailability getTabletAvailability() {
+ if (AccumuloTable.ROOT.tableId().equals(getTableId())
+ || AccumuloTable.METADATA.tableId().equals(getTableId())) {
+ // Override the availability for the system tables
+ return TabletAvailability.HOSTED;
}
+ ensureFetched(ColumnType.AVAILABILITY);
+ return availability;
+ }
+
+ public boolean getHostingRequested() {
+ ensureFetched(ColumnType.HOSTING_REQUESTED);
+ return onDemandHostingRequested;
+ }
+
+ public UnSplittableMetadata getUnSplittable() {
+ ensureFetched(ColumnType.UNSPLITTABLE);
+ return unSplittableMetadata;
}
- public Map<ExternalCompactionId,ExternalCompactionMetadata>
getExternalCompactions() {
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this,
ToStringStyle.SHORT_PREFIX_STYLE).append("tableId", tableId)
+ .append("prevEndRow", prevEndRow).append("sawPrevEndRow",
sawPrevEndRow)
+ .append("endRow", endRow).append("location",
location).append("files", files)
+ .append("scans", scans).append("loadedFiles", loadedFiles)
+ .append("fetchedCols", fetchedCols).append("extent",
extent).append("last", last)
+ .append("suspend", suspend).append("dirName", dirName).append("time",
time)
+ .append("cloned", cloned).append("flush", flush).append("logs", logs)
+ .append("extCompactions", extCompactions).append("availability",
availability)
+ .append("onDemandHostingRequested", onDemandHostingRequested)
+ .append("operationId", operationId).append("selectedFiles",
selectedFiles)
+ .append("futureAndCurrentLocationSet", futureAndCurrentLocationSet)
+ .append("userCompactionsRequested", userCompactionsRequested)
+ .append("unSplittableMetadata", unSplittableMetadata).toString();
+ }
+
- public SortedMap<Key,Value> getKeyValues() {
++ public List<Entry<Key,Value>> getKeyValues() {
+ Preconditions.checkState(keyValues != null, "Requested key values when it
was not saved");
+ return keyValues;
+ }
+
+ public Map<ExternalCompactionId,CompactionMetadata>
getExternalCompactions() {
ensureFetched(ColumnType.ECOMP);
return extCompactions;
}
@@@ -677,21 -577,14 +675,21 @@@
private String dirName;
private MetadataTime time;
private String cloned;
- private ImmutableSortedMap.Builder<Key,Value> keyValues;
+ private ImmutableList.Builder<Entry<Key,Value>> keyValues;
private OptionalLong flush = OptionalLong.empty();
+ private OptionalLong flushNonce = OptionalLong.empty();
private final ImmutableList.Builder<LogEntry> logs =
ImmutableList.builder();
- private OptionalLong compact = OptionalLong.empty();
- private Double splitRatio = null;
- private final ImmutableMap.Builder<ExternalCompactionId,
- ExternalCompactionMetadata> extCompactions = ImmutableMap.builder();
+ private final
ImmutableMap.Builder<ExternalCompactionId,CompactionMetadata> extCompactions =
+ ImmutableMap.builder();
private boolean merged;
+ private TabletAvailability availability = TabletAvailability.ONDEMAND;
+ private boolean onDemandHostingRequested;
+ private TabletOperationId operationId;
+ private boolean futureAndCurrentLocationSet;
+ private final ImmutableSet.Builder<FateId> compacted =
ImmutableSet.builder();
+ private final ImmutableSet.Builder<FateId> userCompactionsRequested =
ImmutableSet.builder();
+ private UnSplittableMetadata unSplittableMetadata;
+ // private Supplier<Long> fileSize;
void table(TableId tableId) {
this.tableId = tableId;
@@@ -776,36 -674,11 +774,36 @@@
this.merged = merged;
}
+ void availability(TabletAvailability availability) {
+ this.availability = availability;
+ }
+
+ void onDemandHostingRequested(boolean onDemandHostingRequested) {
+ this.onDemandHostingRequested = onDemandHostingRequested;
+ }
+
+ void operationId(String val) {
+ Preconditions.checkState(operationId == null);
+ operationId = TabletOperationId.from(val);
+ }
+
+ void compacted(FateId compacted) {
+ this.compacted.add(compacted);
+ }
+
+ void userCompactionsRequested(FateId userCompactionRequested) {
+ this.userCompactionsRequested.add(userCompactionRequested);
+ }
+
+ void unSplittableMetadata(UnSplittableMetadata unSplittableMetadata) {
+ this.unSplittableMetadata = unSplittableMetadata;
+ }
+
- void keyValue(Key key, Value value) {
+ void keyValue(Entry<Key,Value> kv) {
if (this.keyValues == null) {
- this.keyValues = ImmutableSortedMap.naturalOrder();
+ this.keyValues = ImmutableList.builder();
}
- this.keyValues.put(key, value);
+ this.keyValues.add(kv);
}
TabletMetadata build(EnumSet<ColumnType> fetchedCols) {
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
index eb924e7833,0000000000..9bfd2831bf
mode 100644,000000..100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
@@@ -1,333 -1,0 +1,334 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.AVAILABILITY;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FLUSH_ID;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FLUSH_NONCE;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_REQUESTED;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOADED;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGED;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SCANS;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.UNSPLITTABLE;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.USER_COMPACTION_REQUESTED;
+
+import java.util.Arrays;
++import java.util.Collection;
+import java.util.EnumSet;
- import java.util.Set;
++import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.admin.TabletAvailability;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.util.time.SteadyTime;
+import org.apache.hadoop.io.Text;
+
+public class TabletMetadataBuilder implements
Ample.TabletUpdates<TabletMetadataBuilder> {
+
+ public static class InternalBuilder extends
TabletMutatorBase<InternalBuilder> {
+ protected InternalBuilder(KeyExtent extent) {
+ super(extent);
+ }
+
+ @Override
+ public Mutation getMutation() {
+ return super.getMutation();
+ }
+ }
+
+ private final InternalBuilder internalBuilder;
+ EnumSet<TabletMetadata.ColumnType> fetched;
+
+ protected TabletMetadataBuilder(KeyExtent extent) {
+ internalBuilder = new InternalBuilder(extent);
+ fetched = EnumSet.noneOf(TabletMetadata.ColumnType.class);
+ putPrevEndRow(extent.prevEndRow());
+ }
+
+ @Override
+ public TabletMetadataBuilder putPrevEndRow(Text per) {
+ fetched.add(PREV_ROW);
+ internalBuilder.putPrevEndRow(per);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder putFile(ReferencedTabletFile path,
DataFileValue dfv) {
+ fetched.add(FILES);
+ internalBuilder.putFile(path, dfv);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder putFile(StoredTabletFile path, DataFileValue
dfv) {
+ fetched.add(FILES);
+ internalBuilder.putFile(path, dfv);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteFile(StoredTabletFile path) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putScan(StoredTabletFile path) {
+ fetched.add(SCANS);
+ internalBuilder.putScan(path);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteScan(StoredTabletFile path) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putFlushId(long flushId) {
+ fetched.add(FLUSH_ID);
+ internalBuilder.putFlushId(flushId);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder putFlushNonce(long flushNonce) {
+ fetched.add(FLUSH_NONCE);
+ internalBuilder.putFlushId(flushNonce);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder putLocation(TabletMetadata.Location location) {
+ fetched.add(LOCATION);
+ internalBuilder.putLocation(location);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteLocation(TabletMetadata.Location
location) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putDirName(String dirName) {
+ fetched.add(DIR);
+ internalBuilder.putDirName(dirName);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder putWal(LogEntry logEntry) {
+ fetched.add(LOGS);
+ internalBuilder.putWal(logEntry);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteWal(LogEntry logEntry) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putTime(MetadataTime time) {
+ fetched.add(TIME);
+ internalBuilder.putTime(time);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder putBulkFile(ReferencedTabletFile bulkref,
FateId fateId) {
+ fetched.add(LOADED);
+ internalBuilder.putBulkFile(bulkref, fateId);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteBulkFile(StoredTabletFile bulkref) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putSuspension(TServerInstance tserver,
SteadyTime suspensionTime) {
+ fetched.add(SUSPEND);
+ internalBuilder.putSuspension(tserver, suspensionTime);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteSuspension() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putExternalCompaction(ExternalCompactionId
ecid,
+ CompactionMetadata ecMeta) {
+ fetched.add(ECOMP);
+ internalBuilder.putExternalCompaction(ecid, ecMeta);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteExternalCompaction(ExternalCompactionId
ecid) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putCompacted(FateId fateId) {
+ fetched.add(COMPACTED);
+ internalBuilder.putCompacted(fateId);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteCompacted(FateId fateId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putTabletAvailability(TabletAvailability
tabletAvailability) {
+ fetched.add(AVAILABILITY);
+ internalBuilder.putTabletAvailability(tabletAvailability);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder setHostingRequested() {
+ fetched.add(HOSTING_REQUESTED);
+ internalBuilder.setHostingRequested();
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteHostingRequested() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putOperation(TabletOperationId opId) {
+ fetched.add(OPID);
+ internalBuilder.putOperation(opId);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteOperation() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putSelectedFiles(SelectedFiles selectedFiles) {
+ fetched.add(SELECTED);
+ internalBuilder.putSelectedFiles(selectedFiles);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteSelectedFiles() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
- public TabletMetadataBuilder deleteAll(Set<Key> keys) {
++ public TabletMetadataBuilder deleteAll(Collection<Map.Entry<Key,Value>>
entries) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder setMerged() {
+ fetched.add(MERGED);
+ internalBuilder.setMerged();
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteMerged() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder putUserCompactionRequested(FateId fateId) {
+ fetched.add(USER_COMPACTION_REQUESTED);
+ internalBuilder.putUserCompactionRequested(fateId);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteUserCompactionRequested(FateId fateId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder setUnSplittable(UnSplittableMetadata
unSplittableMeta) {
+ fetched.add(UNSPLITTABLE);
+ internalBuilder.setUnSplittable(unSplittableMeta);
+ return this;
+ }
+
+ @Override
+ public TabletMetadataBuilder deleteUnSplittable() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TabletMetadataBuilder automaticallyPutServerLock(boolean b) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @param extraFetched Anything that was put on the builder will
automatically be added to the
+ * fetched set. However, for the case where something was not put
and it needs to be
+ * fetched it can be passed here. For example to simulate a tablet
w/o a location it, no
+ * location will be put and LOCATION would be passed in via this
argument.
+ */
+ public TabletMetadata build(TabletMetadata.ColumnType... extraFetched) {
+ var mutation = internalBuilder.getMutation();
+
+ SortedMap<Key,Value> rowMap = new TreeMap<>();
+ mutation.getUpdates().forEach(cu -> {
+ Key k = new Key(mutation.getRow(), cu.getColumnFamily(),
cu.getColumnQualifier(),
+ cu.getTimestamp());
+ Value v = new Value(cu.getValue());
+ rowMap.put(k, v);
+ });
+
+ fetched.addAll(Arrays.asList(extraFetched));
+
+ return TabletMetadata.convertRow(rowMap.entrySet().iterator(), fetched,
true, false);
+ }
+
+}
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
index cf0208d497,a8b567775a..ab0d6490a3
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
@@@ -16,15 -16,8 +16,16 @@@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.accumulo.server.metadata;
+package org.apache.accumulo.core.metadata.schema;
- import java.util.Set;
++import java.util.Collection;
++import java.util.Map;
+
+import org.apache.accumulo.core.client.admin.TabletAvailability;
+import org.apache.accumulo.core.clientImpl.TabletAvailabilityUtil;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@@ -290,83 -236,6 +291,85 @@@ public abstract class TabletMutatorBase
return mutation;
}
+ @Override
+ public T putTabletAvailability(TabletAvailability tabletAvailability) {
+ TabletColumnFamily.AVAILABILITY_COLUMN.put(mutation,
+ TabletAvailabilityUtil.toValue(tabletAvailability));
+ return getThis();
+ }
+
+ @Override
+ public T setHostingRequested() {
+ TabletColumnFamily.REQUESTED_COLUMN.put(mutation, EMPTY_VALUE);
+ return getThis();
+ }
+
+ @Override
+ public T deleteHostingRequested() {
+ TabletColumnFamily.REQUESTED_COLUMN.putDelete(mutation);
+ return getThis();
+ }
+
+ @Override
- public T deleteAll(Set<Key> keys) {
++ public T deleteAll(Collection<Map.Entry<Key,Value>> entries) {
+ ByteSequence row = new ArrayByteSequence(mutation.getRow());
- keys.forEach(key -> {
++ entries.forEach(e -> {
++ var key = e.getKey();
+ Preconditions.checkArgument(key.getRowData().equals(row), "Unexpected
row %s %s", row, key);
+ Preconditions.checkArgument(key.getColumnVisibilityData().length() == 0,
+ "Non empty column visibility %s", key);
+ });
+
- keys.forEach(key -> {
++ entries.forEach(e -> {
++ var key = e.getKey();
+ mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ });
+
+ return getThis();
+ }
+
+ @Override
+ public T setMerged() {
+ MergedColumnFamily.MERGED_COLUMN.put(mutation,
MergedColumnFamily.MERGED_VALUE);
+ return getThis();
+ }
+
+ @Override
+ public T deleteMerged() {
+ MergedColumnFamily.MERGED_COLUMN.putDelete(mutation);
+ return getThis();
+ }
+
+ @Override
+ public T putUserCompactionRequested(FateId fateId) {
+ mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
fateId.canonical(), "");
+ return getThis();
+ }
+
+ @Override
+ public T deleteUserCompactionRequested(FateId fateId) {
+ mutation.putDelete(UserCompactionRequestedColumnFamily.STR_NAME,
fateId.canonical());
+ return getThis();
+ }
+
+ @Override
+ public T setUnSplittable(UnSplittableMetadata unSplittableMeta) {
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unSplittableMeta.toBase64()));
+ return getThis();
+ }
+
+ @Override
+ public T deleteUnSplittable() {
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.putDelete(mutation);
+ return getThis();
+ }
+
+ @Override
+ public T automaticallyPutServerLock(boolean b) {
+ putServerLock = b;
+ return getThis();
+ }
+
public void setCloseAfterMutate(AutoCloseable closeable) {
this.closeAfterMutate = closeable;
}
diff --cc
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index 0f51d61283,8c3d84de2c..84bb870f99
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -53,13 -51,10 +54,14 @@@ import java.util.TreeMap
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
++import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.accumulo.core.client.admin.TabletAvailability;
+import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@@ -173,11 -160,11 +175,13 @@@ public class TabletMetadataTest
assertEquals(extent, tm.getExtent());
assertEquals(Set.of(tf1, tf2), Set.copyOf(tm.getFiles()));
assertEquals(Map.of(tf1, dfv1, tf2, dfv2), tm.getFilesMap());
+
assertEquals(tm.getFilesMap().values().stream().mapToLong(DataFileValue::getSize).sum(),
+ tm.getFileSize());
assertEquals(6L, tm.getFlushId().getAsLong());
- assertEquals(rowMap, tm.getKeyValues());
- TreeMap<Key,Value> actualRowMap = new TreeMap<>();
- tm.getKeyValues().forEach(entry -> actualRowMap.put(entry.getKey(),
entry.getValue()));
++ SortedMap<Key,Value> actualRowMap = tm.getKeyValues().stream().collect(
++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b,
TreeMap::new));
+ assertEquals(rowMap, actualRowMap);
- assertEquals(Map.of(new StoredTabletFile(bf1), 56L, new
StoredTabletFile(bf2), 59L),
+ assertEquals(Map.of(new StoredTabletFile(bf1), fateId1, new
StoredTabletFile(bf2), fateId2),
tm.getLoaded());
assertEquals(HostAndPort.fromParts("server1", 8555),
tm.getLocation().getHostAndPort());
assertEquals("s001", tm.getLocation().getSession());
@@@ -407,10 -390,8 +411,10 @@@
b.file(stf, new DataFileValue(0, 0, 0));
b.log(LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()));
b.scan(stf);
- b.loadedFile(stf, 0L);
+ b.loadedFile(stf, FateId.from(FateInstanceType.USER, UUID.randomUUID()));
+ b.compacted(FateId.from(FateInstanceType.USER, UUID.randomUUID()));
+ b.userCompactionsRequested(FateId.from(FateInstanceType.USER,
UUID.randomUUID()));
- b.keyValue(new Key(), new Value());
+ b.keyValue(new AbstractMap.SimpleImmutableEntry<>(new Key(), new
Value()));
var tm2 = b.build(EnumSet.allOf(ColumnType.class));
assertEquals(1, tm2.getExternalCompactions().size());
@@@ -427,143 -408,10 +431,142 @@@
assertEquals(1, tm2.getScans().size());
assertThrows(UnsupportedOperationException.class, () ->
tm2.getScans().add(stf));
assertEquals(1, tm2.getLoaded().size());
- assertThrows(UnsupportedOperationException.class, () ->
tm2.getLoaded().put(stf, 0L));
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm2.getLoaded().put(stf, FateId.from(FateInstanceType.USER,
UUID.randomUUID())));
assertEquals(1, tm2.getKeyValues().size());
- assertThrows(UnsupportedOperationException.class,
- () -> tm2.getKeyValues().put(new Key(), new Value()));
+ assertThrows(UnsupportedOperationException.class, () ->
tm2.getKeyValues().remove(null));
+ assertEquals(1, tm2.getCompacted().size());
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm2.getCompacted().add(FateId.from(FateInstanceType.USER,
UUID.randomUUID())));
+ assertEquals(1, tm2.getUserCompactionsRequested().size());
+ assertThrows(UnsupportedOperationException.class, () ->
tm2.getUserCompactionsRequested()
+ .add(FateId.from(FateInstanceType.USER, UUID.randomUUID())));
+ }
+
+ @Test
+ public void testCompactionRequestedColumn() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+ FateInstanceType type = FateInstanceType.fromTableId(extent.tableId());
+ FateId userCompactFateId1 = FateId.from(type, UUID.randomUUID());
+ FateId userCompactFateId2 = FateId.from(type, UUID.randomUUID());
+
+ // Test column set
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId1.canonical(), "");
+ mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId2.canonical(), "");
+
+ TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(USER_COMPACTION_REQUESTED), true, false);
+ assertEquals(2, tm.getUserCompactionsRequested().size());
+ assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId1));
+ assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId2));
+
+ // Column not set
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(USER_COMPACTION_REQUESTED), true, false);
+ assertTrue(tm.getUserCompactionsRequested().isEmpty());
+ // Column not fetched
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(ColumnType.PREV_ROW), true, false);
+ assertThrows(IllegalStateException.class,
tm::getUserCompactionsRequested);
+ }
+
+ @Test
+ public void testUnsplittableColumn() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ StoredTabletFile sf1 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"));
+ StoredTabletFile sf2 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf2.rf"));
+ StoredTabletFile sf3 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf3.rf"));
+ // Same path as sf4 but with a range
+ StoredTabletFile sf4 =
+ StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf3.rf"), new Range("a", "b"));
+
+ // Test with files
+ var unsplittableMeta1 =
+ UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1, sf2, sf3));
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta1.toBase64()));
+ TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta1, tm.getUnSplittable(), true);
+
+ // Test empty file set
+ var unsplittableMeta2 = UnSplittableMetadata.toUnSplittable(extent, 100,
110, 120, Set.of());
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta2.toBase64()));
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta2, tm.getUnSplittable(), true);
+
+ // Make sure not equals works as well
+ assertUnsplittable(unsplittableMeta1, unsplittableMeta2, false);
+
+ // Test with ranges
+ // use sf4 which includes sf4 instead of sf3 which has a range
+ var unsplittableMeta3 =
+ UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1, sf2, sf4));
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta3.toBase64()));
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta3, tm.getUnSplittable(), true);
+
+ // make sure not equals when all the file paths are equal but one has a
range
+ assertUnsplittable(unsplittableMeta1, unsplittableMeta3, false);
+
+ // Column not set
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertNull(tm.getUnSplittable());
+
+ // Column not fetched
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(ColumnType.PREV_ROW), true, false);
+ assertThrows(IllegalStateException.class, tm::getUnSplittable);
+ }
+
+ @Test
+ public void testUnsplittableWithRange() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ // Files with same path and different ranges
+ StoredTabletFile sf1 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"));
+ StoredTabletFile sf2 =
+ StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"), new Range("a", "b"));
+ StoredTabletFile sf3 =
+ StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"), new Range("a", "d"));
+
+ var meta1 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1));
+ var meta2 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf2));
+ var meta3 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf3));
+
+ // compare each against the others to make sure not equal
+ assertUnsplittable(meta1, meta2, false);
+ assertUnsplittable(meta1, meta3, false);
+ assertUnsplittable(meta2, meta3, false);
+ }
+
+ private void assertUnsplittable(UnSplittableMetadata meta1,
UnSplittableMetadata meta2,
+ boolean equal) {
+ assertEquals(equal, meta1.equals(meta2));
+ assertEquals(equal, meta1.hashCode() == meta2.hashCode());
+ assertEquals(equal, meta1.toBase64().equals(meta2.toBase64()));
+ }
+
+ @Test
+ public void testUnknownColFamily() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+
+ mutation.put("1234567890abcdefg", "xyz", "v1");
+ assertThrows(IllegalStateException.class, () -> TabletMetadata
+ .convertRow(toRowMap(mutation).entrySet().iterator(),
EnumSet.of(MERGED), true, false));
}
private SortedMap<Key,Value> toRowMap(Mutation mutation) {
diff --cc
server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index ed8a835e34,e53c908907..e767bca510
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -233,13 -389,21 +233,13 @@@ public class MetadataTableUtil
return new Pair<>(result, sizes);
}
- public static void removeUnusedWALEntries(ServerContext context, KeyExtent
extent,
- final List<LogEntry> entries, ServiceLock zooLock) {
- TabletMutator tablet = context.getAmple().mutateTablet(extent);
- entries.forEach(tablet::deleteWal);
- tablet.putZooLock(zooLock);
- tablet.mutate();
- }
-
private static Mutation createCloneMutation(TableId srcTableId, TableId
tableId,
- Map<Key,Value> tablet) {
+ Iterable<Entry<Key,Value>> tablet) {
- KeyExtent ke =
KeyExtent.fromMetaRow(tablet.keySet().iterator().next().getRow());
+ KeyExtent ke =
KeyExtent.fromMetaRow(tablet.iterator().next().getKey().getRow());
Mutation m = new Mutation(TabletsSection.encodeRow(tableId, ke.endRow()));
- for (Entry<Key,Value> entry : tablet.entrySet()) {
+ for (Entry<Key,Value> entry : tablet) {
if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME))
{
String cf = entry.getKey().getColumnQualifier().toString();
if (!cf.startsWith("../") && !cf.contains(":")) {
@@@ -346,48 -523,33 +346,48 @@@
if (!cloneSuccessful) {
srcFiles.addAll(srcTablet.getFiles());
}
- }
- if (cloneSuccessful) {
- continue;
- }
+ while (cmp > 0) {
+ srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+ srcEndRow = srcTablet.getEndRow();
+ cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0) {
+ throw new TabletDeletedException(
+ "Tablets deleted from src during clone : " + cloneEndRow + "
" + srcEndRow);
+ }
- if (srcFiles.containsAll(cloneFiles)) {
- // write out marker that this tablet was successfully cloned
- Mutation m = new Mutation(cloneTablet.getExtent().toMetaRow());
- m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK"));
- bw.addMutation(m);
- } else {
- // delete existing cloned tablet entry
- Mutation m = new Mutation(cloneTablet.getExtent().toMetaRow());
+ if (!cloneSuccessful) {
+ srcFiles.addAll(srcTablet.getFiles());
+ }
+ }
- for (Entry<Key,Value> entry : cloneTablet.getKeyValues()) {
- Key k = entry.getKey();
- m.putDelete(k.getColumnFamily(), k.getColumnQualifier(),
k.getTimestamp());
+ if (cloneSuccessful) {
+ continue;
}
- bw.addMutation(m);
+ if (srcFiles.containsAll(cloneFiles)) {
+ // write out marker that this tablet was successfully cloned
+ Mutation m = new Mutation(cloneTablet.getExtent().toMetaRow());
+ m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK"));
+ bw.addMutation(m);
+ } else {
+ // delete existing cloned tablet entry
+ Mutation m = new Mutation(cloneTablet.getExtent().toMetaRow());
- for (Entry<Key,Value> entry :
cloneTablet.getKeyValues().entrySet()) {
- for (TabletMetadata st : srcTablets) {
- bw.addMutation(createCloneMutation(srcTableId, tableId,
st.getKeyValues()));
- }
++ for (Entry<Key,Value> entry : cloneTablet.getKeyValues()) {
+ Key k = entry.getKey();
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(),
k.getTimestamp());
+ }
+
+ bw.addMutation(m);
- rewrites++;
+ for (TabletMetadata st : srcTablets) {
+ bw.addMutation(createCloneMutation(srcTableId, tableId,
st.getKeyValues()));
+ }
+
+ rewrites++;
+ }
}
}
diff --cc
server/base/src/test/java/org/apache/accumulo/server/SetEncodingIteratorTest.java
index 90195493cd,0000000000..fcf4f14ab4
mode 100644,000000..100644
---
a/server/base/src/test/java/org/apache/accumulo/server/SetEncodingIteratorTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/SetEncodingIteratorTest.java
@@@ -1,204 -1,0 +1,209 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
++import java.util.stream.Collectors;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.server.metadata.iterators.SetEncodingIterator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class SetEncodingIteratorTest {
+
+ private TabletMetadata tmOneFile;
+ private TabletMetadata tmMultipleFiles;
+ private SetEncodingIterator setEqualityIterator;
+ private SetEncodingIterator setEqualityIteratorNoFiles;
+ private SetEncodingIterator setEqualityIteratorOneFile;
+ private SortedMapIterator sortedMapIterator;
+ private SortedMapIterator sortedMapIteratorNoFiles;
+ private SortedMapIterator sortedMapIteratorOneFile;
+
+ private KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"),
new Text("da"));
+
+ private StoredTabletFile file1 =
+ new ReferencedTabletFile(new
Path("dfs://nn1/acc/tables/1/t-0001/sf1.rf")).insert();
+ private StoredTabletFile file2 =
+ new ReferencedTabletFile(new
Path("dfs://nn1/acc/tables/1/t-0001/sf2.rf")).insert();
+ private StoredTabletFile file3 =
+ new ReferencedTabletFile(new
Path("dfs://nn1/acc/tables/1/t-0001/sf3.rf")).insert();
+
+ @BeforeEach
+ public void setUp() throws IOException {
+
+ // Create tablet metadata with no files
+ TabletMetadata tmNoFiles =
TabletMetadata.builder(extent).putFlushId(7).build();
+
+ // Create tablet metadata with one file
+ StoredTabletFile singleFile =
+ new ReferencedTabletFile(new
Path("dfs://nn1/acc/tables/1/t-0001/sf1.rf")).insert();
+ tmOneFile = TabletMetadata.builder(extent).putFile(singleFile, new
DataFileValue(100, 50))
+ .putFlushId(8).build();
+
+ // Create tablet metadata with multiple files
+ tmMultipleFiles = TabletMetadata.builder(extent).putFile(file1, new
DataFileValue(0, 0))
+ .putFile(file2, new DataFileValue(555, 23)).putFile(file3, new
DataFileValue(234, 13))
+ .putFlushId(6).build();
+
+ var extent2 = new KeyExtent(extent.tableId(), null, extent.endRow());
+ // create another tablet metadata using extent2 w/ diff files and add it
to sortedMap. This
+ // will add another row to the test data which ensures that iterator does
not go to another row.
+ StoredTabletFile file4 =
+ new ReferencedTabletFile(new
Path("dfs://nn1/acc/tables/1/t-0002/sf4.rf")).insert();
+ StoredTabletFile file5 =
+ new ReferencedTabletFile(new
Path("dfs://nn1/acc/tables/1/t-0002/sf5.rf")).insert();
+ StoredTabletFile file6 =
+ new ReferencedTabletFile(new
Path("dfs://nn1/acc/tables/1/t-0002/sf6.rf")).insert();
+ TabletMetadata tmMultipleFiles2 = TabletMetadata.builder(extent2)
+ .putFile(file4, new DataFileValue(100, 50)).putFile(file5, new
DataFileValue(200, 75))
+ .putFile(file6, new DataFileValue(300, 100)).putFlushId(7).build();
+
+ // Convert TabletMetadata to a SortedMap
- SortedMap<Key,Value> sortedMapNoFiles = new
TreeMap<>(tmNoFiles.getKeyValues());
- SortedMap<Key,Value> sortedMapOneFile = new
TreeMap<>(tmOneFile.getKeyValues());
- SortedMap<Key,Value> sortedMap = new
TreeMap<>(tmMultipleFiles.getKeyValues());
- SortedMap<Key,Value> sortedMap2 = new
TreeMap<>(tmMultipleFiles2.getKeyValues());
++ SortedMap<Key,Value> sortedMapNoFiles =
tmNoFiles.getKeyValues().stream().collect(
++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a3, b3) ->
b3, TreeMap::new));
++ SortedMap<Key,Value> sortedMapOneFile =
tmOneFile.getKeyValues().stream().collect(
++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a2, b2) ->
b2, TreeMap::new));
++ SortedMap<Key,Value> sortedMap =
tmMultipleFiles.getKeyValues().stream().collect(
++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a1, b1) ->
b1, TreeMap::new));
++ SortedMap<Key,Value> sortedMap2 =
tmMultipleFiles2.getKeyValues().stream().collect(
++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b,
TreeMap::new));
+ // Add the second tablet metadata to the sortedMap
+ sortedMap.putAll(sortedMap2);
+
+ // Create a SortedMapIterator using the SortedMap
+ sortedMapIterator = new SortedMapIterator(sortedMap);
+ sortedMapIteratorNoFiles = new SortedMapIterator(sortedMapNoFiles);
+ sortedMapIteratorOneFile = new SortedMapIterator(sortedMapOneFile);
+
+ // Set the SortedMapIterator as the source for SetEqualityIterator
+ setEqualityIterator = new SetEncodingIterator();
+ setEqualityIterator.init(sortedMapIterator,
Map.of(SetEncodingIterator.CONCAT_VALUE, "true"),
+ null);
+ setEqualityIteratorNoFiles = new SetEncodingIterator();
+ setEqualityIteratorNoFiles.init(sortedMapIteratorNoFiles,
+ Map.of(SetEncodingIterator.CONCAT_VALUE, "false"), null);
+ setEqualityIteratorOneFile = new SetEncodingIterator();
+ setEqualityIteratorOneFile.init(sortedMapIteratorOneFile,
+ Map.of(SetEncodingIterator.CONCAT_VALUE, "true"), null);
+ }
+
+ @Test
+ public void testTabletWithNoFiles() throws IOException {
+ // Creating a test range
+ Text tabletRow = new Text(extent.toMetaRow());
+ Text family = MetadataSchema.TabletsSection.DataFileColumnFamily.NAME;
+
+ Range range = Range.exact(tabletRow, family);
+
+ // Invoking the seek method
+ setEqualityIteratorNoFiles.seek(range, Collections.emptyList(), false);
+
+ // Asserting the result
+ assertEquals(new Key(tabletRow, family),
setEqualityIteratorNoFiles.getTopKey());
+ // The iterator should produce a value that is equal to the expected
value on the condition
+ var condition =
SetEncodingIterator.createCondition(Collections.emptySet(),
+ storedTabletFile -> ((StoredTabletFile)
storedTabletFile).getMetadata().getBytes(UTF_8),
+ family);
+ assertArrayEquals(condition.getValue().toArray(),
+ setEqualityIteratorNoFiles.getTopValue().get());
+ }
+
+ @Test
+ public void testTabletWithOneFile() throws IOException {
+ // Creating a test range
+ Text tabletRow = new Text(extent.toMetaRow());
+ Text family = MetadataSchema.TabletsSection.DataFileColumnFamily.NAME;
+
+ Range range = Range.exact(tabletRow, family);
+
+ // Invoking the seek method
+ setEqualityIteratorOneFile.seek(range, Collections.emptyList(), false);
+
+ // Asserting the result
+ assertEquals(new Key(tabletRow, family),
setEqualityIteratorOneFile.getTopKey());
+ // The iterator should produce a value that is equal to the expected
value on the condition
+ var condition =
SetEncodingIterator.createConditionWithVal(tmOneFile.getFilesMap().entrySet(),
+ entry -> new Pair<>(entry.getKey().getMetadata().getBytes(UTF_8),
+ entry.getValue().encode()),
+ family);
+ assertArrayEquals(condition.getValue().toArray(),
+ setEqualityIteratorOneFile.getTopValue().get());
+ }
+
+ @Test
+ public void testTabletWithMultipleFiles() throws IOException {
+ // Creating a test range
+ Text tabletRow = new Text(extent.toMetaRow());
+ Text family = MetadataSchema.TabletsSection.DataFileColumnFamily.NAME;
+
+ Range range = Range.exact(tabletRow, family);
+
+ // Invoking the seek method
+ setEqualityIterator.seek(range, Collections.emptyList(), false);
+
+ // Asserting the result
+ assertEquals(new Key(tabletRow, family), setEqualityIterator.getTopKey());
+ // The iterator should produce a value that is equal to the expected
value on the condition
+ var condition =
+
SetEncodingIterator.createConditionWithVal(tmMultipleFiles.getFilesMap().entrySet(),
+ entry -> new Pair<>(entry.getKey().getMetadata().getBytes(UTF_8),
+ entry.getValue().encode()),
+ family);
+ assertArrayEquals(condition.getValue().toArray(),
setEqualityIterator.getTopValue().get());
+
+ }
+
+ @Test
+ public void testInvalidConcatValueOption() throws IOException {
+ SetEncodingIterator iter = new SetEncodingIterator();
+ iter.init(null, Map.of(SetEncodingIterator.CONCAT_VALUE, "true"), null);
+ iter.init(null, Map.of(SetEncodingIterator.CONCAT_VALUE, "false"), null);
+ assertThrows(IllegalArgumentException.class, () -> iter.init(null,
Map.of(), null));
+ assertThrows(IllegalArgumentException.class,
+ () -> iter.init(null, Map.of(SetEncodingIterator.CONCAT_VALUE,
"yes"), null));
+ assertThrows(IllegalArgumentException.class,
+ () -> iter.init(null, Map.of(SetEncodingIterator.CONCAT_VALUE, ""),
null));
+
+ }
+
+}
diff --cc
server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
index 207c1d0a01,0000000000..8347353f60
mode 100644,000000..100644
---
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
@@@ -1,194 -1,0 +1,203 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.manager.state;
+
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
++import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.UUID;
++import java.util.stream.Collectors;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.manager.state.TabletManagement;
+import
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LastLocationColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
++import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.jupiter.api.Test;
+
+public class TabletManagementTest {
+
+ private SortedMap<Key,Value> toRowMap(Mutation mutation) {
+ SortedMap<Key,Value> rowMap = new TreeMap<>();
+ mutation.getUpdates().forEach(cu -> {
+ Key k = new Key(mutation.getRow(), cu.getColumnFamily(),
cu.getColumnQualifier(),
+ cu.getTimestamp());
+ Value v = new Value(cu.getValue());
+ rowMap.put(k, v);
+ });
+ return rowMap;
+ }
+
+ private SortedMap<Key,Value> createMetadataEntryKV(KeyExtent extent) {
+
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+
+ FateInstanceType type = FateInstanceType.fromTableId(extent.tableId());
+ FateId fateId1 = FateId.from(type, UUID.randomUUID());
+ FateId fateId2 = FateId.from(type, UUID.randomUUID());
+
+ DIRECTORY_COLUMN.put(mutation, new Value("t-0001757"));
+ FLUSH_COLUMN.put(mutation, new Value("6"));
+ TIME_COLUMN.put(mutation, new Value("M123456789"));
+
+ StoredTabletFile bf1 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/bf1")).insert();
+ StoredTabletFile bf2 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/bf2")).insert();
+
mutation.at().family(BulkFileColumnFamily.NAME).qualifier(bf1.getMetadata())
+ .put(fateId1.canonical());
+
mutation.at().family(BulkFileColumnFamily.NAME).qualifier(bf2.getMetadata())
+ .put(fateId2.canonical());
+
+ mutation.at().family(ClonedColumnFamily.NAME).qualifier("").put("OK");
+
+ DataFileValue dfv1 = new DataFileValue(555, 23);
+ StoredTabletFile tf1 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/df1.rf")).insert();
+ StoredTabletFile tf2 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/df2.rf")).insert();
+
mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf1.getMetadata()).put(dfv1.encode());
+ DataFileValue dfv2 = new DataFileValue(234, 13);
+
mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf2.getMetadata()).put(dfv2.encode());
+
+
mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555");
+
+
mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555");
+
+ LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID());
+ le1.addToMutation(mutation);
+ LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID());
+ le2.addToMutation(mutation);
+
+ StoredTabletFile sf1 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf")).insert();
+ StoredTabletFile sf2 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf2.rf")).insert();
+
mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf1.getMetadata()).put("");
+
mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf2.getMetadata()).put("");
+
+ return toRowMap(mutation);
+
+ }
+
+ @Test
+ public void testEncodeDecodeWithReasons() throws Exception {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ final Set<ManagementAction> actions =
+ Set.of(ManagementAction.NEEDS_LOCATION_UPDATE,
ManagementAction.NEEDS_SPLITTING);
+
+ final SortedMap<Key,Value> entries = createMetadataEntryKV(extent);
+
+ TabletManagement.addActions(entries::put, entries.firstKey().getRow(),
actions);
+ Key key = entries.firstKey();
+ Value val = WholeRowIterator.encodeRow(new ArrayList<>(entries.keySet()),
+ new ArrayList<>(entries.values()));
+
+ // Remove the REASONS column from the entries map for the comparison check
+ // below
+ entries.remove(new Key(key.getRow().toString(), "REASONS", ""));
+
+ TabletManagement tmi = new TabletManagement(key, val, true);
- assertEquals(entries, tmi.getTabletMetadata().getKeyValues());
++ TabletMetadata tabletMetadata = tmi.getTabletMetadata();
++ assertEquals(entries, tabletMetadata.getKeyValues().stream().collect(
++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b,
TreeMap::new)));
+ assertEquals(actions, tmi.getActions());
+ }
+
+ @Test
+ public void testEncodeDecodeWithErrors() throws Exception {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ final SortedMap<Key,Value> entries = createMetadataEntryKV(extent);
+
+ TabletManagement.addError(entries::put, entries.firstKey().getRow(),
+ new UnsupportedOperationException("Not supported."));
+ Key key = entries.firstKey();
+ Value val = WholeRowIterator.encodeRow(new ArrayList<>(entries.keySet()),
+ new ArrayList<>(entries.values()));
+
+ // Remove the ERROR column from the entries map for the comparison check
+ // below
+ entries.remove(new Key(key.getRow().toString(), "ERROR", ""));
+
+ TabletManagement tmi = new TabletManagement(key, val, true);
- assertEquals(entries, tmi.getTabletMetadata().getKeyValues());
++ TabletMetadata tabletMetadata = tmi.getTabletMetadata();
++ assertEquals(entries, tabletMetadata.getKeyValues().stream().collect(
++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b,
TreeMap::new)));
+ assertEquals("Not supported.", tmi.getErrorMessage());
+ }
+
+ @Test
+ public void testBinary() throws Exception {
+ // test end row with non ascii data
+ Text endRow = new Text(new byte[] {'m', (byte) 0xff});
+ KeyExtent extent = new KeyExtent(TableId.of("5"), endRow, new Text("da"));
+
+ final Set<ManagementAction> actions =
+ Set.of(ManagementAction.NEEDS_LOCATION_UPDATE,
ManagementAction.NEEDS_SPLITTING);
+
+ final SortedMap<Key,Value> entries = createMetadataEntryKV(extent);
+
+ TabletManagement.addActions(entries::put, entries.firstKey().getRow(),
actions);
+ Key key = entries.firstKey();
+ Value val = WholeRowIterator.encodeRow(new ArrayList<>(entries.keySet()),
+ new ArrayList<>(entries.values()));
+
+ assertTrue(entries.keySet().stream().allMatch(k ->
k.getRow().equals(extent.toMetaRow())));
+
+ // Remove the REASONS column from the entries map for the comparison check
+ // below
+ entries.remove(new Key(key.getRow(), new Text("REASONS"), new Text("")));
+
+ TabletManagement tmi = new TabletManagement(key, val, true);
- assertEquals(entries, tmi.getTabletMetadata().getKeyValues());
++ TabletMetadata tabletMetadata = tmi.getTabletMetadata();
++ assertEquals(entries, tabletMetadata.getKeyValues().stream().collect(
++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b,
TreeMap::new)));
+ assertEquals(actions, tmi.getActions());
+
+ }
+}
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
index 3cb48a68b0,0000000000..3f98f812fb
mode 100644,000000..100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
@@@ -1,128 -1,0 +1,128 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.tableOps.merge;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.TabletOperationId;
+import org.apache.accumulo.core.metadata.schema.TabletOperationType;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.manager.Manager;
+import org.apache.accumulo.manager.tableOps.ManagerRepo;
+import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Delete tablets that were merged into another tablet.
+ */
+public class DeleteTablets extends ManagerRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private final MergeInfo data;
+
+ private final byte[] lastTabletEndRow;
+
+ private static final Logger log =
LoggerFactory.getLogger(DeleteTablets.class);
+
+ DeleteTablets(MergeInfo mergeInfo, Text lastTabletEndRow) {
+ this.data = mergeInfo;
+ this.lastTabletEndRow = lastTabletEndRow == null ? null :
TextUtil.getBytes(lastTabletEndRow);
+ }
+
+ @Override
+ public Repo<Manager> call(FateId fateId, Manager manager) throws Exception {
+
+ KeyExtent range = data.getMergeExtent();
+ log.debug("{} Deleting tablets for {}", fateId, range);
+ var opid = TabletOperationId.from(TabletOperationType.MERGING, fateId);
+
+ AtomicLong acceptedCount = new AtomicLong();
+ AtomicLong rejectedCount = new AtomicLong();
+ // delete tablets
+ Consumer<Ample.ConditionalResult> resultConsumer = result -> {
+ if (result.getStatus() == Ample.ConditionalResult.Status.ACCEPTED) {
+ acceptedCount.incrementAndGet();
+ } else {
+ log.error("{} failed to update {}", fateId, result.getExtent());
+ rejectedCount.incrementAndGet();
+ }
+ };
+
+ long submitted = 0;
+
+ try (
+ var tabletsMetadata =
+
manager.getContext().getAmple().readTablets().forTable(range.tableId())
+ .overlapping(range.prevEndRow(),
range.endRow()).saveKeyValues().build();
+ var tabletsMutator =
+
manager.getContext().getAmple().conditionallyMutateTablets(resultConsumer)) {
+
+ var lastEndRow = lastTabletEndRow == null ? null : new
Text(lastTabletEndRow);
+
+ for (var tabletMeta : tabletsMetadata) {
+ MergeTablets.validateTablet(tabletMeta, fateId, opid, data.tableId);
+
+ var tabletMutator =
tabletsMutator.mutateTablet(tabletMeta.getExtent())
+
.requireOperation(opid).requireAbsentLocation().requireAbsentLogs();
+
+ // do not delete the last tablet
+ if (Objects.equals(tabletMeta.getExtent().endRow(), lastEndRow)) {
+ // Clear the merged marker after we are finished on the last tablet
+ tabletMutator.deleteMerged();
+ tabletMutator.submit((tm) -> !tm.hasMerged());
+ submitted++;
+ break;
+ }
+
+ if (log.isTraceEnabled()) {
- tabletMeta.getKeyValues().keySet().forEach(key -> {
- log.trace("{} deleting {}", fateId, key);
++ tabletMeta.getKeyValues().forEach(entry -> {
++ log.trace("{} deleting {}", fateId, entry.getKey());
+ });
+ }
+
- tabletMutator.deleteAll(tabletMeta.getKeyValues().keySet());
++ tabletMutator.deleteAll(tabletMeta.getKeyValues());
+
+ // the entire tablet is being deleted, so do not want to add lock
entry to the tablet
+ tabletMutator.automaticallyPutServerLock(false);
+
+ // if the tablet no longer exists, then it was successful
+ tabletMutator.submit(Ample.RejectionHandler.acceptAbsentTablet());
+ submitted++;
+ }
+ }
+
+ Preconditions.checkState(acceptedCount.get() == submitted &&
rejectedCount.get() == 0,
+ "Failed to delete tablets accepted:%s != %s rejected:%s",
acceptedCount.get(), submitted,
+ rejectedCount.get());
+
+ log.debug("{} deleted {} tablets", fateId, submitted);
+
+ return new FinishTableRangeOp(data);
+ }
+}
diff --cc test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
index 1a711ba8d9,8169037e99..a25b6c749e
--- a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
@@@ -37,14 -30,6 +37,15 @@@ import java.util.List
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
+import java.util.Set;
++import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
@@@ -397,59 -280,4 +398,61 @@@ public class SplitIT extends AccumuloCl
assertEquals(1000, c.createScanner(tableName).stream().count());
}
}
+
+ /**
+ * Test attempting to split a tablet that has an unexpected column
+ */
+ @Test
+ public void testUnexpectedColumn() throws Exception {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ var ctx = getServerContext();
+ var tableId = ctx.getTableId(tableName);
+ var extent = new KeyExtent(tableId, null, null);
+
+ // This column is not expected to be present
+ var tabletMutator = ctx.getAmple().mutateTablet(extent);
+ tabletMutator.setMerged();
+ tabletMutator.mutate();
+
+ var tabletMetadata =
ctx.getAmple().readTablets().forTable(tableId).saveKeyValues().build()
+ .stream().collect(MoreCollectors.onlyElement());
+ assertEquals(extent, tabletMetadata.getExtent());
+
+ // remove the srv:lock column for tests as this will change
+ // because we are changing the metadata from the IT.
- var original = new TreeMap<>(tabletMetadata.getKeyValues());
++ var original = (SortedMap<Key,Value>)
tabletMetadata.getKeyValues().stream()
++ .collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a1, b1)
-> b1, TreeMap::new));
+ assertTrue(original.keySet().removeIf(LOCK_COLUMN::hasColumns));
+
+ // Split operation should fail because of the unexpected column.
+ var splits = new TreeSet<>(List.of(new Text("m")));
+ assertThrows(AccumuloException.class, () ->
c.tableOperations().addSplits(tableName, splits));
+ assertEquals(Set.of(),
Set.copyOf(c.tableOperations().listSplits(tableName)));
+
+ // The tablet should be left in a bad state, so simulate a manual
cleanup
+ var tabletMetadata2 =
ctx.getAmple().readTablets().forTable(tableId).saveKeyValues().build()
+ .stream().collect(MoreCollectors.onlyElement());
+ assertEquals(extent, tabletMetadata.getExtent());
+
+ // tablet should have an operation id set, but nothing else changed
- var kvCopy = new TreeMap<>(tabletMetadata2.getKeyValues());
++ var kvCopy = (SortedMap<Key,Value>)
tabletMetadata2.getKeyValues().stream()
++ .collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) ->
b, TreeMap::new));
+ assertTrue(kvCopy.keySet().removeIf(LOCK_COLUMN::hasColumns));
+ assertTrue(kvCopy.keySet().removeIf(OPID_COLUMN::hasColumns));
+ assertEquals(original, kvCopy);
+
+ // remove the offending columns
+ tabletMutator = ctx.getAmple().mutateTablet(extent);
+ tabletMutator.deleteMerged();
+ tabletMutator.deleteOperation();
+ tabletMutator.mutate();
+
+ // after cleaning up the tablet metadata, should be able to split
+ c.tableOperations().addSplits(tableName, splits);
+ assertEquals(Set.of(new Text("m")),
Set.copyOf(c.tableOperations().listSplits(tableName)));
+ }
+ }
}
diff --cc test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
index 402ed1ff50,7667f84207..a82f4eddc5
--- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
@@@ -103,14 -96,15 +103,16 @@@ public class MetricsIT extends Configur
@Test
public void confirmMetricsPublished() throws Exception {
- doWorkToGenerateMetrics();
- cluster.stop();
-
Set<String> unexpectedMetrics = Set.of(METRICS_COMPACTOR_MAJC_STUCK,
METRICS_SCAN_YIELDS);
+ // add sserver as flaky until scan server included in mini tests.
- Set<String> flakyMetrics = Set.of(METRICS_FATE_TYPE_IN_PROGRESS,
METRICS_GC_WAL_ERRORS,
- METRICS_SCAN_BUSY_TIMEOUT_COUNTER,
METRICS_SCAN_RESERVATION_CONFLICT_COUNTER,
+ Set<String> flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS,
METRICS_FATE_TYPE_IN_PROGRESS,
+ METRICS_TSERVER_TABLETS_ONLINE_ONDEMAND,
METRICS_TSERVER_TABLETS_ONDEMAND_UNLOADED_FOR_MEM,
+ METRICS_COMPACTOR_MAJC_STUCK, METRICS_MANAGER_ROOT_TGW_ERRORS,
+ METRICS_MANAGER_META_TGW_ERRORS, METRICS_MANAGER_USER_TGW_ERRORS,
+ METRICS_SCAN_TABLET_METADATA_CACHE, METRICS_SCAN_BUSY_TIMEOUT_COUNTER,
METRICS_SCAN_RESERVATION_TOTAL_TIMER,
METRICS_SCAN_RESERVATION_WRITEOUT_TIMER,
- METRICS_SERVER_IDLE);
++ METRICS_SERVER_IDLE, METRICS_SCAN_RESERVATION_CONFLICT_COUNTER,
METRICS_GC_WAL_ERRORS,
+ METRICS_SCAN_TABLET_METADATA_CACHE);
Map<String,String> expectedMetricNames = this.getMetricFields();
flakyMetrics.forEach(expectedMetricNames::remove); // might not see these
