This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 861d0fdfc2b7bdd6da72e0e5652c027643ec5a49
Merge: 1c706c541e 1ae2b17e2b
Author: Christopher L. Shannon <cshan...@apache.org>
AuthorDate: Sat Apr 6 16:11:28 2024 -0400
Merge branch 'main' into elasticity
.../core/metadata/schema/TabletMetadata.java | 389 ++++++++++++++-------
.../core/metadata/schema/TabletMetadataTest.java | 105 +++++-
2 files changed, 362 insertions(+), 132 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index be7d2fb056,f2ad719fd8..6f2ffd6237
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -92,44 -82,63 +92,80 @@@ import com.google.common.collect.Immuta
import com.google.common.net.HostAndPort;
public class TabletMetadata {
+
private static final Logger log =
LoggerFactory.getLogger(TabletMetadata.class);
- private TableId tableId;
- private Text prevEndRow;
- private boolean sawPrevEndRow = false;
- private Text endRow;
- private Location location;
- private Map<StoredTabletFile,DataFileValue> files;
- private List<StoredTabletFile> scans;
- private Map<StoredTabletFile,FateId> loadedFiles;
- private SelectedFiles selectedFiles;
- private EnumSet<ColumnType> fetchedCols;
- private KeyExtent extent;
- private Location last;
- private SuspendingTServer suspend;
- private String dirName;
- private MetadataTime time;
- private String cloned;
- private SortedMap<Key,Value> keyValues;
- private OptionalLong flush = OptionalLong.empty();
- private OptionalLong flushNonce = OptionalLong.empty();
- private List<LogEntry> logs;
- private Map<ExternalCompactionId,CompactionMetadata> extCompactions;
- private boolean merged;
- private TabletAvailability availability = TabletAvailability.ONDEMAND;
- private boolean onDemandHostingRequested = false;
- private TabletOperationId operationId;
- private boolean futureAndCurrentLocationSet = false;
- private Set<FateId> compacted;
- private Set<FateId> userCompactionsRequested;
- private UnSplittableMetadata unSplittableMetadata;
- private Supplier<Long> fileSize;
+ private final TableId tableId;
+ private final Text prevEndRow;
+ private final boolean sawPrevEndRow;
- private final Text oldPrevEndRow;
- private final boolean sawOldPrevEndRow;
+ private final Text endRow;
+ private final Location location;
+ private final Map<StoredTabletFile,DataFileValue> files;
+ private final List<StoredTabletFile> scans;
- private final Map<StoredTabletFile,Long> loadedFiles;
++ private final Map<StoredTabletFile,FateId> loadedFiles;
++ private final SelectedFiles selectedFiles;
+ private final EnumSet<ColumnType> fetchedCols;
+ private final Supplier<KeyExtent> extent;
+ private final Location last;
+ private final SuspendingTServer suspend;
+ private final String dirName;
+ private final MetadataTime time;
+ private final String cloned;
+ private final SortedMap<Key,Value> keyValues;
+ private final OptionalLong flush;
++ private final OptionalLong flushNonce;
+ private final List<LogEntry> logs;
- private final OptionalLong compact;
- private final Double splitRatio;
- private final Map<ExternalCompactionId,ExternalCompactionMetadata>
extCompactions;
++ private final Map<ExternalCompactionId,CompactionMetadata> extCompactions;
+ private final boolean merged;
++ private final TabletAvailability availability;
++ private final boolean onDemandHostingRequested;
++ private final TabletOperationId operationId;
++ private final boolean futureAndCurrentLocationSet;
++ private final Set<FateId> compacted;
++ private final Set<FateId> userCompactionsRequested;
++ private final UnSplittableMetadata unSplittableMetadata;
++ private final Supplier<Long> fileSize;
+
+ private TabletMetadata(Builder tmBuilder) {
+ this.tableId = tmBuilder.tableId;
+ this.prevEndRow = tmBuilder.prevEndRow;
+ this.sawPrevEndRow = tmBuilder.sawPrevEndRow;
- this.oldPrevEndRow = tmBuilder.oldPrevEndRow;
- this.sawOldPrevEndRow = tmBuilder.sawOldPrevEndRow;
+ this.endRow = tmBuilder.endRow;
+ this.location = tmBuilder.location;
+ this.files = Objects.requireNonNull(tmBuilder.files.build());
+ this.scans = Objects.requireNonNull(tmBuilder.scans.build());
+ this.loadedFiles = tmBuilder.loadedFiles.build();
++ this.selectedFiles = tmBuilder.selectedFiles;
+ this.fetchedCols = Objects.requireNonNull(tmBuilder.fetchedCols);
+ this.last = tmBuilder.last;
+ this.suspend = tmBuilder.suspend;
+ this.dirName = tmBuilder.dirName;
+ this.time = tmBuilder.time;
+ this.cloned = tmBuilder.cloned;
+ this.keyValues =
Optional.ofNullable(tmBuilder.keyValues).map(ImmutableSortedMap.Builder::build)
+ .orElse(null);
+ this.flush = tmBuilder.flush;
++ this.flushNonce = tmBuilder.flushNonce;
+ this.logs = Objects.requireNonNull(tmBuilder.logs.build());
- this.compact = Objects.requireNonNull(tmBuilder.compact);
- this.splitRatio = tmBuilder.splitRatio;
+ this.extCompactions =
Objects.requireNonNull(tmBuilder.extCompactions.build());
+ this.merged = tmBuilder.merged;
++ this.availability = Objects.requireNonNull(tmBuilder.availability);
++ this.onDemandHostingRequested = tmBuilder.onDemandHostingRequested;
++ this.operationId = tmBuilder.operationId;
++ this.futureAndCurrentLocationSet = tmBuilder.futureAndCurrentLocationSet;
++ this.compacted = tmBuilder.compacted.build();
++ this.userCompactionsRequested =
tmBuilder.userCompactionsRequested.build();
++ this.unSplittableMetadata = tmBuilder.unSplittableMetadata;
++ this.fileSize =
++ Suppliers.memoize(() ->
files.values().stream().mapToLong(DataFileValue::getSize).sum());
+ this.extent =
+ Suppliers.memoize(() -> new KeyExtent(getTableId(), getEndRow(),
getPrevEndRow()));
+ }
+ public static TabletMetadataBuilder builder(KeyExtent extent) {
+ return new TabletMetadataBuilder(extent);
+ }
+
public enum LocationType {
CURRENT, FUTURE, LAST
}
@@@ -427,40 -414,12 +460,31 @@@
return extCompactions;
}
+ public Set<FateId> getCompacted() {
+ ensureFetched(ColumnType.COMPACTED);
+ return compacted;
+ }
+
+ /**
+ * @return the operation id if it exists, null otherwise
+ * @see MetadataSchema.TabletsSection.ServerColumnFamily#OPID_COLUMN
+ */
+ public TabletOperationId getOperationId() {
+ ensureFetched(ColumnType.OPID);
+ return operationId;
+ }
+
+ public boolean isFutureAndCurrentLocationSet() {
+ return futureAndCurrentLocationSet;
+ }
+
@VisibleForTesting
public static <E extends Entry<Key,Value>> TabletMetadata
convertRow(Iterator<E> rowIter,
- EnumSet<ColumnType> fetchedColumns, boolean buildKeyValueMap) {
+ EnumSet<ColumnType> fetchedColumns, boolean buildKeyValueMap, boolean
suppressLocationError) {
Objects.requireNonNull(rowIter);
- TabletMetadata te = new TabletMetadata();
- final ImmutableSortedMap.Builder<Key,Value> kvBuilder =
- buildKeyValueMap ? ImmutableSortedMap.naturalOrder() : null;
-
- final var filesBuilder =
ImmutableMap.<StoredTabletFile,DataFileValue>builder();
- final var scansBuilder = ImmutableList.<StoredTabletFile>builder();
- final var logsBuilder = ImmutableList.<LogEntry>builder();
- final var extCompBuilder =
ImmutableMap.<ExternalCompactionId,CompactionMetadata>builder();
- final var loadedFilesBuilder =
ImmutableMap.<StoredTabletFile,FateId>builder();
- final var compactedBuilder = ImmutableSet.<FateId>builder();
- final var userCompactionsRequestedBuilder =
ImmutableSet.<FateId>builder();
+ final var tmBuilder = new Builder();
++
ByteSequence row = null;
while (rowIter.hasNext()) {
@@@ -488,17 -447,16 +512,17 @@@
case TabletColumnFamily.STR_NAME:
switch (qual) {
case PREV_ROW_QUAL:
- te.prevEndRow =
TabletColumnFamily.decodePrevEndRow(kv.getValue());
- te.sawPrevEndRow = true;
+
tmBuilder.prevEndRow(TabletColumnFamily.decodePrevEndRow(kv.getValue()));
+ tmBuilder.sawPrevEndRow(true);
break;
- case OLD_PREV_ROW_QUAL:
-
tmBuilder.oldPrevEndRow(TabletColumnFamily.decodePrevEndRow(kv.getValue()));
- tmBuilder.sawOldPrevEndRow(true);
+ case AVAILABILITY_QUAL:
- te.availability =
TabletAvailabilityUtil.fromValue(kv.getValue());
++
tmBuilder.availability(TabletAvailabilityUtil.fromValue(kv.getValue()));
break;
- case SPLIT_RATIO_QUAL:
- tmBuilder.splitRatio(Double.parseDouble(val));
+ case REQUESTED_QUAL:
- te.onDemandHostingRequested = true;
++ tmBuilder.onDemandHostingRequested(true);
break;
+ default:
+ throw new IllegalStateException("Unexpected TabletColumnFamily
qualifier: " + qual);
}
break;
case ServerColumnFamily.STR_NAME:
@@@ -506,22 -464,16 +530,22 @@@
case DIRECTORY_QUAL:
Preconditions.checkArgument(ServerColumnFamily.isValidDirCol(val),
"Saw invalid dir name %s %s", key, val);
- te.dirName = val;
+ tmBuilder.dirName(val);
break;
case TIME_QUAL:
- te.time = MetadataTime.parse(val);
+ tmBuilder.time(MetadataTime.parse(val));
break;
case FLUSH_QUAL:
- te.flush = OptionalLong.of(Long.parseLong(val));
+ tmBuilder.flush(Long.parseLong(val));
break;
- case COMPACT_QUAL:
- tmBuilder.compact(Long.parseLong(val));
+ case FLUSH_NONCE_QUAL:
- te.flushNonce = OptionalLong.of(Long.parseUnsignedLong(val,
16));
++ tmBuilder.flushNonce(Long.parseUnsignedLong(val, 16));
+ break;
+ case OPID_QUAL:
- te.setOperationIdOnce(val);
++ tmBuilder.operationId(val);
+ break;
+ case SELECTED_QUAL:
- te.selectedFiles = SelectedFiles.from(val);
++ tmBuilder.selectedFiles(SelectedFiles.from(val));
break;
}
break;
@@@ -533,103 -485,39 +557,58 @@@
BulkFileColumnFamily.getBulkLoadTid(val));
break;
case CurrentLocationColumnFamily.STR_NAME:
- te.setLocationOnce(val, qual, LocationType.CURRENT,
suppressLocationError);
- tmBuilder.location(val, qual, LocationType.CURRENT);
++ tmBuilder.location(val, qual, LocationType.CURRENT,
suppressLocationError);
break;
case FutureLocationColumnFamily.STR_NAME:
- te.setLocationOnce(val, qual, LocationType.FUTURE,
suppressLocationError);
- tmBuilder.location(val, qual, LocationType.FUTURE);
++ tmBuilder.location(val, qual, LocationType.FUTURE,
suppressLocationError);
break;
case LastLocationColumnFamily.STR_NAME:
- te.last = Location.last(val, qual);
+ tmBuilder.last(Location.last(val, qual));
break;
case SuspendLocationColumn.STR_NAME:
- te.suspend = SuspendingTServer.fromValue(kv.getValue());
+ tmBuilder.suspend(SuspendingTServer.fromValue(kv.getValue()));
break;
case ScanFileColumnFamily.STR_NAME:
- scansBuilder.add(new StoredTabletFile(qual));
+ tmBuilder.scan(new StoredTabletFile(qual));
break;
case ClonedColumnFamily.STR_NAME:
- te.cloned = val;
+ tmBuilder.cloned(val);
break;
case LogColumnFamily.STR_NAME:
- logsBuilder.add(LogEntry.fromMetaWalEntry(kv));
+ tmBuilder.log(LogEntry.fromMetaWalEntry(kv));
break;
case ExternalCompactionColumnFamily.STR_NAME:
- extCompBuilder.put(ExternalCompactionId.of(qual),
CompactionMetadata.fromJson(val));
- tmBuilder.extCompaction(ExternalCompactionId.of(qual),
- ExternalCompactionMetadata.fromJson(val));
++ tmBuilder.extCompaction(ExternalCompactionId.of(qual),
CompactionMetadata.fromJson(val));
break;
case MergedColumnFamily.STR_NAME:
- te.merged = true;
+ tmBuilder.merged(true);
break;
+ case CompactedColumnFamily.STR_NAME:
- compactedBuilder.add(FateId.from(qual));
++ tmBuilder.compacted(FateId.from(qual));
+ break;
+ case UserCompactionRequestedColumnFamily.STR_NAME:
- userCompactionsRequestedBuilder.add(FateId.from(qual));
++ tmBuilder.userCompactionsRequested(FateId.from(qual));
+ break;
+ case SplitColumnFamily.STR_NAME:
+ if (qual.equals(SplitColumnFamily.UNSPLITTABLE_QUAL)) {
- te.unSplittableMetadata =
UnSplittableMetadata.toUnSplittable(val);
++
tmBuilder.unSplittableMetadata(UnSplittableMetadata.toUnSplittable(val));
+ } else {
+ throw new IllegalStateException("Unexpected SplitColumnFamily
qualifier: " + qual);
+ }
+ break;
default:
throw new IllegalStateException("Unexpected family " + fam);
+
}
}
- if (AccumuloTable.ROOT.tableId().equals(te.tableId)
- || AccumuloTable.METADATA.tableId().equals(te.tableId)) {
++ if (AccumuloTable.ROOT.tableId().equals(tmBuilder.tableId)
++ || AccumuloTable.METADATA.tableId().equals(tmBuilder.tableId)) {
+ // Override the availability for the system tables
- te.availability = TabletAvailability.HOSTED;
++ tmBuilder.availability(TabletAvailability.HOSTED);
+ }
+
- var files = filesBuilder.build();
- te.files = files;
- te.fileSize =
- Suppliers.memoize(() ->
files.values().stream().mapToLong(DataFileValue::getSize).sum());
- te.loadedFiles = loadedFilesBuilder.build();
- te.fetchedCols = fetchedColumns;
- te.scans = scansBuilder.build();
- te.logs = logsBuilder.build();
- te.extCompactions = extCompBuilder.build();
- te.compacted = compactedBuilder.build();
- te.userCompactionsRequested = userCompactionsRequestedBuilder.build();
- if (buildKeyValueMap) {
- te.keyValues = kvBuilder.build();
- }
- return te;
- }
-
- /**
- * Sets a location only once.
- *
- * @param val server to set for Location object
- * @param qual session to set for Location object
- * @param lt location type to use to construct Location object
- * @param suppressError set to true to suppress an exception being thrown,
else false
- * @throws IllegalStateException if an operation id or location is already
set
- */
- private void setLocationOnce(String val, String qual, LocationType lt,
boolean suppressError) {
- if (location != null) {
- if (!suppressError) {
- throw new IllegalStateException("Attempted to set second location for
tableId: " + tableId
- + " endrow: " + endRow + " -- " + location + " " + qual + " " +
val);
- }
- futureAndCurrentLocationSet = true;
- }
- location = new Location(val, qual, lt);
- }
-
- /**
- * Sets an operation ID only once.
- *
- * @param val operation id to set
- * @throws IllegalStateException if an operation id or location is already
set
- */
- private void setOperationIdOnce(String val) {
- Preconditions.checkState(operationId == null);
- operationId = TabletOperationId.from(val);
+ return tmBuilder.build(fetchedColumns);
}
@VisibleForTesting
@@@ -673,4 -560,132 +651,159 @@@
.map(sld -> sld.getAddress(ServiceLockData.ThriftService.TSERV))
.map(address -> new TServerInstance(address,
stat.getEphemeralOwner()));
}
+
+ static class Builder {
+ private TableId tableId;
+ private Text prevEndRow;
+ private boolean sawPrevEndRow;
- private Text oldPrevEndRow;
- private boolean sawOldPrevEndRow;
+ private Text endRow;
+ private Location location;
+ private final ImmutableMap.Builder<StoredTabletFile,DataFileValue> files =
+ ImmutableMap.builder();
+ private final ImmutableList.Builder<StoredTabletFile> scans =
ImmutableList.builder();
- private final ImmutableMap.Builder<StoredTabletFile,Long> loadedFiles =
ImmutableMap.builder();
++ private final ImmutableMap.Builder<StoredTabletFile,FateId> loadedFiles =
++ ImmutableMap.builder();
++ private SelectedFiles selectedFiles;
+ private EnumSet<ColumnType> fetchedCols;
+ private Location last;
+ private SuspendingTServer suspend;
+ private String dirName;
+ private MetadataTime time;
+ private String cloned;
+ private ImmutableSortedMap.Builder<Key,Value> keyValues;
+ private OptionalLong flush = OptionalLong.empty();
++ private OptionalLong flushNonce = OptionalLong.empty();
+ private final ImmutableList.Builder<LogEntry> logs =
ImmutableList.builder();
- private OptionalLong compact = OptionalLong.empty();
- private Double splitRatio = null;
- private final ImmutableMap.Builder<ExternalCompactionId,
- ExternalCompactionMetadata> extCompactions = ImmutableMap.builder();
++ private final
ImmutableMap.Builder<ExternalCompactionId,CompactionMetadata> extCompactions =
++ ImmutableMap.builder();
+ private boolean merged;
++ private TabletAvailability availability = TabletAvailability.ONDEMAND;
++ private boolean onDemandHostingRequested;
++ private TabletOperationId operationId;
++ private boolean futureAndCurrentLocationSet;
++ private final ImmutableSet.Builder<FateId> compacted =
ImmutableSet.builder();
++ private final ImmutableSet.Builder<FateId> userCompactionsRequested =
ImmutableSet.builder();
++ private UnSplittableMetadata unSplittableMetadata;
++ // private Supplier<Long> fileSize;
+
+ void table(TableId tableId) {
+ this.tableId = tableId;
+ }
+
+ void endRow(Text endRow) {
+ this.endRow = endRow;
+ }
+
+ void prevEndRow(Text prevEndRow) {
+ this.prevEndRow = prevEndRow;
+ }
+
+ void sawPrevEndRow(boolean sawPrevEndRow) {
+ this.sawPrevEndRow = sawPrevEndRow;
+ }
+
- void oldPrevEndRow(Text oldPrevEndRow) {
- this.oldPrevEndRow = oldPrevEndRow;
- }
-
- void sawOldPrevEndRow(boolean sawOldPrevEndRow) {
- this.sawOldPrevEndRow = sawOldPrevEndRow;
- }
-
- void splitRatio(Double splitRatio) {
- this.splitRatio = splitRatio;
- }
-
+ void dirName(String dirName) {
+ this.dirName = dirName;
+ }
+
+ void time(MetadataTime time) {
+ this.time = time;
+ }
+
+ void flush(long flush) {
+ this.flush = OptionalLong.of(flush);
+ }
+
- void compact(long compact) {
- this.compact = OptionalLong.of(compact);
++ void flushNonce(long flushNonce) {
++ this.flushNonce = OptionalLong.of(flushNonce);
+ }
+
+ void file(StoredTabletFile stf, DataFileValue dfv) {
+ this.files.put(stf, dfv);
+ }
+
- void loadedFile(StoredTabletFile stf, Long tid) {
- this.loadedFiles.put(stf, tid);
++ void loadedFile(StoredTabletFile stf, FateId fateId) {
++ this.loadedFiles.put(stf, fateId);
+ }
+
- void location(String val, String qual, LocationType lt) {
++ void selectedFiles(SelectedFiles selectedFiles) {
++ this.selectedFiles = selectedFiles;
++ }
++
++ void location(String val, String qual, LocationType lt, boolean
suppressError) {
+ if (location != null) {
- throw new IllegalStateException("Attempted to set second location for
tableId: " + tableId
- + " endrow: " + endRow + " -- " + location + " " + qual + " " +
val);
++ if (!suppressError) {
++ throw new IllegalStateException("Attempted to set second location
for tableId: " + tableId
++ + " endrow: " + endRow + " -- " + location + " " + qual + " " +
val);
++ }
++ futureAndCurrentLocationSet = true;
+ }
- this.location = new Location(val, qual, lt);
++ location = new Location(val, qual, lt);
+ }
+
+ void last(Location last) {
+ this.last = last;
+ }
+
+ void suspend(SuspendingTServer suspend) {
+ this.suspend = suspend;
+ }
+
+ void scan(StoredTabletFile stf) {
+ this.scans.add(stf);
+ }
+
+ void cloned(String cloned) {
+ this.cloned = cloned;
+ }
+
+ void log(LogEntry log) {
+ this.logs.add(log);
+ }
+
- void extCompaction(ExternalCompactionId id, ExternalCompactionMetadata
metadata) {
++ void extCompaction(ExternalCompactionId id, CompactionMetadata metadata) {
+ this.extCompactions.put(id, metadata);
+ }
+
+ void merged(boolean merged) {
+ this.merged = merged;
+ }
+
++ void availability(TabletAvailability availability) {
++ this.availability = availability;
++ }
++
++ void onDemandHostingRequested(boolean onDemandHostingRequested) {
++ this.onDemandHostingRequested = onDemandHostingRequested;
++ }
++
++ void operationId(String val) {
++ Preconditions.checkState(operationId == null);
++ operationId = TabletOperationId.from(val);
++ }
++
++ void compacted(FateId compacted) {
++ this.compacted.add(compacted);
++ }
++
++ void userCompactionsRequested(FateId userCompactionRequested) {
++ this.userCompactionsRequested.add(userCompactionRequested);
++ }
++
++ void unSplittableMetadata(UnSplittableMetadata unSplittableMetadata) {
++ this.unSplittableMetadata = unSplittableMetadata;
++ }
++
+ void keyValue(Key key, Value value) {
+ if (this.keyValues == null) {
+ this.keyValues = ImmutableSortedMap.naturalOrder();
+ }
+ this.keyValues.put(key, value);
+ }
+
+ TabletMetadata build(EnumSet<ColumnType> fetchedCols) {
+ this.fetchedCols = fetchedCols;
+ return new TabletMetadata(this);
+ }
+ }
}
diff --cc
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index eda2f2f2ba,ac94fc8f69..c2f6a19e69
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -22,18 -22,17 +22,20 @@@ import static java.util.stream.Collecto
import static org.apache.accumulo.core.metadata.StoredTabletFile.serialize;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_VALUE;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
++import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_NONCE_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
+ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn.SUSPEND_COLUMN;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.AVAILABILITY;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_REQUESTED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.UNSPLITTABLE;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.USER_COMPACTION_REQUESTED;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
@@@ -74,16 -70,14 +77,16 @@@ import org.apache.accumulo.core.metadat
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LastLocationColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SplitColumnFamily;
- import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily;
+ import org.apache.accumulo.core.metadata.schema.TabletMetadata.Builder;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
-import org.apache.accumulo.core.spi.compaction.CompactionExecutorId;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.accumulo.core.spi.compaction.CompactorGroupId;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
-import org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.Test;
@@@ -135,12 -126,25 +138,28 @@@ public class TabletMetadataTest
mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf2.getMetadata()).put("");
MERGED_COLUMN.put(mutation, new Value());
+ FateId userCompactFateId = FateId.from(type, UUID.randomUUID());
+ mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId.canonical(), "");
+ var unsplittableMeta =
+ UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1, sf2));
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta.toBase64()));
- OLD_PREV_ROW_COLUMN.put(mutation, TabletColumnFamily.encodePrevEndRow(new
Text("oldPrev")));
+ long suspensionTime = System.currentTimeMillis();
+ TServerInstance ser1 = new
TServerInstance(HostAndPort.fromParts("server1", 8555), "s001");
+ Value suspend = SuspendingTServer.toValue(ser1, suspensionTime);
+ SUSPEND_COLUMN.put(mutation, suspend);
- double splitRatio = .3;
- SPLIT_RATIO_COLUMN.put(mutation, new Value(Double.toString(splitRatio)));
++ FLUSH_NONCE_COLUMN.put(mutation, new Value(Long.toHexString(10L)));
+
+ ExternalCompactionId ecid =
ExternalCompactionId.generate(UUID.randomUUID());
+ ReferencedTabletFile tmpFile =
+ ReferencedTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
- CompactionExecutorId ceid = CompactionExecutorIdImpl.externalId("G1");
+ Set<StoredTabletFile> jobFiles =
+ Set.of(StoredTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/b2.rf")));
- ExternalCompactionMetadata ecMeta = new
ExternalCompactionMetadata(jobFiles, jobFiles, tmpFile,
- "localhost:4444", CompactionKind.SYSTEM, (short) 2, ceid, false,
false, 44L);
++ CompactionMetadata ecMeta =
++ new CompactionMetadata(jobFiles, tmpFile, "cid1",
CompactionKind.USER, (short) 3,
++ CompactorGroupId.of("Q1"), true,
FateId.from(FateInstanceType.USER, UUID.randomUUID()));
+ mutation.put(ExternalCompactionColumnFamily.STR_NAME, ecid.canonical(),
ecMeta.toJson());
+
SortedMap<Key,Value> rowMap = toRowMap(mutation);
TabletMetadata tm =
TabletMetadata.convertRow(rowMap.entrySet().iterator(),
@@@ -172,8 -175,11 +191,10 @@@
assertEquals("M123456789", tm.getTime().encode());
assertEquals(Set.of(sf1, sf2), Set.copyOf(tm.getScans()));
assertTrue(tm.hasMerged());
- assertEquals(new Text("oldPrev"), tm.getOldPrevEndRow());
- assertTrue(tm.sawOldPrevEndRow());
- assertEquals(SuspendingTServer.fromValue(suspend), tm.getSuspend());
- assertEquals(splitRatio, tm.getSplitRatio());
+ assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId));
+ assertEquals(unsplittableMeta, tm.getUnSplittable());
+ assertEquals(ecMeta.toJson(),
tm.getExternalCompactions().get(ecid).toJson());
++ assertEquals(10, tm.getFlushNonce().getAsLong());
}
@Test
@@@ -281,14 -288,13 +302,13 @@@
// test SUSPENDED
mutation = TabletColumnFamily.createPrevRowMutation(extent);
-
mutation.at().family(SuspendLocationColumn.SUSPEND_COLUMN.getColumnFamily())
- .qualifier(SuspendLocationColumn.SUSPEND_COLUMN.getColumnQualifier())
- .put(SuspendingTServer.toValue(ser2, 1000L));
+ mutation.at().family(SUSPEND_COLUMN.getColumnFamily())
+
.qualifier(SUSPEND_COLUMN.getColumnQualifier()).put(SuspendingTServer.toValue(ser2,
1000L));
rowMap = toRowMap(mutation);
- tm = TabletMetadata.convertRow(rowMap.entrySet().iterator(), colsToFetch,
false);
+ tm = TabletMetadata.convertRow(rowMap.entrySet().iterator(), colsToFetch,
false, false);
- assertEquals(TabletState.SUSPENDED, tm.getTabletState(tservers));
+ assertEquals(TabletState.SUSPENDED, TabletState.compute(tm, tservers));
assertEquals(1000L, tm.getSuspend().suspensionTime);
assertEquals(ser2.getHostAndPort(), tm.getSuspend().server);
assertNull(tm.getLocation());
@@@ -340,130 -346,67 +360,207 @@@
assertTrue(closeCalled.get());
}
+ @Test
+ public void testTmBuilderImmutable() {
+ TabletMetadata.Builder b = new Builder();
+ var tm = b.build(EnumSet.allOf(ColumnType.class));
+
+ ExternalCompactionId ecid =
ExternalCompactionId.generate(UUID.randomUUID());
+ ReferencedTabletFile tmpFile =
+ ReferencedTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
- CompactionExecutorId ceid = CompactionExecutorIdImpl.externalId("G1");
+ StoredTabletFile stf = StoredTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/b2.rf"));
- Set<StoredTabletFile> jobFiles = Set.of(stf);
- ExternalCompactionMetadata ecMeta = new
ExternalCompactionMetadata(jobFiles, jobFiles, tmpFile,
- "localhost:4444", CompactionKind.SYSTEM, (short) 2, ceid, false,
false, 44L);
++ CompactionMetadata ecMeta =
++ new CompactionMetadata(Set.of(stf), tmpFile, "cid1",
CompactionKind.USER, (short) 3,
++ CompactorGroupId.of("Q1"), true,
FateId.from(FateInstanceType.USER, UUID.randomUUID()));
+
+ // Verify the various collections are immutable and non-null (except for
getKeyValues) if
+ // nothing set on the builder
+ assertTrue(tm.getExternalCompactions().isEmpty());
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm.getExternalCompactions().put(ecid, ecMeta));
+ assertTrue(tm.getFiles().isEmpty());
+ assertTrue(tm.getFilesMap().isEmpty());
+ assertThrows(UnsupportedOperationException.class, () ->
tm.getFiles().add(stf));
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm.getFilesMap().put(stf, new DataFileValue(0, 0, 0)));
+ assertTrue(tm.getLogs().isEmpty());
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm.getLogs().add(LogEntry.fromPath("localhost+8020/" +
UUID.randomUUID())));
+ assertTrue(tm.getScans().isEmpty());
+ assertThrows(UnsupportedOperationException.class, () ->
tm.getScans().add(stf));
+ assertTrue(tm.getLoaded().isEmpty());
- assertThrows(UnsupportedOperationException.class, () ->
tm.getLoaded().put(stf, 0L));
++ assertThrows(UnsupportedOperationException.class,
++ () -> tm.getLoaded().put(stf, FateId.from(FateInstanceType.USER,
UUID.randomUUID())));
+ assertThrows(IllegalStateException.class, tm::getKeyValues);
++ assertTrue(tm.getCompacted().isEmpty());
++ assertThrows(UnsupportedOperationException.class,
++ () -> tm.getCompacted().add(FateId.from(FateInstanceType.USER,
UUID.randomUUID())));
++ assertTrue(tm.getUserCompactionsRequested().isEmpty());
++ assertThrows(UnsupportedOperationException.class, () ->
tm.getUserCompactionsRequested()
++ .add(FateId.from(FateInstanceType.USER, UUID.randomUUID())));
+
+ // Set some data in the collections and very they are not empty but still
immutable
+ b.extCompaction(ecid, ecMeta);
+ b.file(stf, new DataFileValue(0, 0, 0));
+ b.log(LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()));
+ b.scan(stf);
- b.loadedFile(stf, 0L);
++ b.loadedFile(stf, FateId.from(FateInstanceType.USER, UUID.randomUUID()));
++ b.compacted(FateId.from(FateInstanceType.USER, UUID.randomUUID()));
++ b.userCompactionsRequested(FateId.from(FateInstanceType.USER,
UUID.randomUUID()));
+ b.keyValue(new Key(), new Value());
+ var tm2 = b.build(EnumSet.allOf(ColumnType.class));
+
+ assertEquals(1, tm2.getExternalCompactions().size());
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm2.getExternalCompactions().put(ecid, ecMeta));
+ assertEquals(1, tm2.getFiles().size());
+ assertEquals(1, tm2.getFilesMap().size());
+ assertThrows(UnsupportedOperationException.class, () ->
tm2.getFiles().add(stf));
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm2.getFilesMap().put(stf, new DataFileValue(0, 0, 0)));
+ assertEquals(1, tm2.getLogs().size());
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm2.getLogs().add(LogEntry.fromPath("localhost+8020/" +
UUID.randomUUID())));
+ assertEquals(1, tm2.getScans().size());
+ assertThrows(UnsupportedOperationException.class, () ->
tm2.getScans().add(stf));
+ assertEquals(1, tm2.getLoaded().size());
- assertThrows(UnsupportedOperationException.class, () ->
tm2.getLoaded().put(stf, 0L));
++ assertThrows(UnsupportedOperationException.class,
++ () -> tm2.getLoaded().put(stf, FateId.from(FateInstanceType.USER,
UUID.randomUUID())));
+ assertEquals(1, tm2.getKeyValues().size());
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm2.getKeyValues().put(new Key(), new Value()));
++ assertEquals(1, tm2.getCompacted().size());
++ assertThrows(UnsupportedOperationException.class,
++ () -> tm2.getCompacted().add(FateId.from(FateInstanceType.USER,
UUID.randomUUID())));
++ assertEquals(1, tm2.getUserCompactionsRequested().size());
++ assertThrows(UnsupportedOperationException.class, () ->
tm2.getUserCompactionsRequested()
++ .add(FateId.from(FateInstanceType.USER, UUID.randomUUID())));
++ }
++
+ @Test
+ public void testCompactionRequestedColumn() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+ FateInstanceType type = FateInstanceType.fromTableId(extent.tableId());
+ FateId userCompactFateId1 = FateId.from(type, UUID.randomUUID());
+ FateId userCompactFateId2 = FateId.from(type, UUID.randomUUID());
+
+ // Test column set
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId1.canonical(), "");
+ mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId2.canonical(), "");
+
+ TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(USER_COMPACTION_REQUESTED), true, false);
+ assertEquals(2, tm.getUserCompactionsRequested().size());
+ assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId1));
+ assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId2));
+
+ // Column not set
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(USER_COMPACTION_REQUESTED), true, false);
+ assertTrue(tm.getUserCompactionsRequested().isEmpty());
+ // Column not fetched
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(ColumnType.PREV_ROW), true, false);
+ assertThrows(IllegalStateException.class,
tm::getUserCompactionsRequested);
+ }
+
+ @Test
+ public void testUnsplittableColumn() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ StoredTabletFile sf1 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"));
+ StoredTabletFile sf2 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf2.rf"));
+ StoredTabletFile sf3 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf3.rf"));
+ // Same path as sf4 but with a range
+ StoredTabletFile sf4 =
+ StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf3.rf"), new Range("a", "b"));
+
+ // Test with files
+ var unsplittableMeta1 =
+ UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1, sf2, sf3));
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta1.toBase64()));
+ TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta1, tm.getUnSplittable(), true);
+
+ // Test empty file set
+ var unsplittableMeta2 = UnSplittableMetadata.toUnSplittable(extent, 100,
110, 120, Set.of());
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta2.toBase64()));
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta2, tm.getUnSplittable(), true);
+
+ // Make sure not equals works as well
+ assertUnsplittable(unsplittableMeta1, unsplittableMeta2, false);
+
+ // Test with ranges
+ // use sf4 which includes sf4 instead of sf3 which has a range
+ var unsplittableMeta3 =
+ UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1, sf2, sf4));
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta3.toBase64()));
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta3, tm.getUnSplittable(), true);
+
+ // make sure not equals when all the file paths are equal but one has a
range
+ assertUnsplittable(unsplittableMeta1, unsplittableMeta3, false);
+
+ // Column not set
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertNull(tm.getUnSplittable());
+
+ // Column not fetched
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(ColumnType.PREV_ROW), true, false);
+ assertThrows(IllegalStateException.class, tm::getUnSplittable);
+ }
+
+ @Test
+ public void testUnsplittableWithRange() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ // Files with same path and different ranges
+ StoredTabletFile sf1 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"));
+ StoredTabletFile sf2 =
+ StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"), new Range("a", "b"));
+ StoredTabletFile sf3 =
+ StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"), new Range("a", "d"));
+
+ var meta1 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1));
+ var meta2 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf2));
+ var meta3 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf3));
+
+ // compare each against the others to make sure not equal
+ assertUnsplittable(meta1, meta2, false);
+ assertUnsplittable(meta1, meta3, false);
+ assertUnsplittable(meta2, meta3, false);
+ }
+
+ private void assertUnsplittable(UnSplittableMetadata meta1,
UnSplittableMetadata meta2,
+ boolean equal) {
+ assertEquals(equal, meta1.equals(meta2));
+ assertEquals(equal, meta1.hashCode() == meta2.hashCode());
+ assertEquals(equal, meta1.toBase64().equals(meta2.toBase64()));
+ }
+
+ @Test
+ public void testUnknownColFamily() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+
+ mutation.put("1234567890abcdefg", "xyz", "v1");
+ assertThrows(IllegalStateException.class, () -> TabletMetadata
+ .convertRow(toRowMap(mutation).entrySet().iterator(),
EnumSet.of(MERGED), true, false));
}
private SortedMap<Key,Value> toRowMap(Mutation mutation) {
