This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 7cf496a5fd Removes race conditions between tablet refresh and minor
compaction (#3634)
7cf496a5fd is described below
commit 7cf496a5fd3d00f52468f32f21ead91573f64284
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Fri Jul 28 12:55:58 2023 -0400
Removes race conditions between tablet refresh and minor compaction (#3634)
This commit moved the tablet twoards using a TabletMetadata object as
the source of the tablets files instead of tracking the files itself.
The move twoards using TabletMetadata support the changes to fix the
race conditions with tablet refresh. Committing a minor compaction to
walogs and the metadata table is now mutually exclusive with refreshing
a tablets metadata. Scans should always see a consistent view of the in
memory maps a tablet has and the files it has after this change. Before
this change a tablet scan could see the file produced by a minor
compaction and the in memory map that was the source of that file.
Scans should only see one or the other.
fixes #3625
---
.../org/apache/accumulo/core/conf/Property.java | 2 -
.../apache/accumulo/core/logging/TabletLogger.java | 4 +-
.../accumulo/core/metadata/StoredTabletFile.java | 1 -
.../core/tabletserver/thrift/TTabletRefresh.java | 566 ---------------------
.../thrift/TabletServerClientService.java | 286 +++++------
core/src/main/thrift/tabletserver.thrift | 7 +-
.../org/apache/accumulo/server/fs/VolumeUtil.java | 2 +
.../accumulo/server/util/MetadataTableUtil.java | 3 +-
.../coordinator/CompactionCoordinator.java | 12 +-
.../manager/tableOps/bulkVer2/TabletRefresher.java | 49 +-
.../apache/accumulo/tserver/AssignmentHandler.java | 4 +-
.../accumulo/tserver/TabletClientHandler.java | 21 +-
.../org/apache/accumulo/tserver/TabletServer.java | 61 +--
.../accumulo/tserver/tablet/DatafileManager.java | 287 -----------
.../tserver/tablet/MetadataUpdateCount.java | 84 ---
.../accumulo/tserver/tablet/ScanfileManager.java | 139 +++++
.../accumulo/tserver/tablet/SnapshotTablet.java | 2 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 371 +++++++-------
.../apache/accumulo/tserver/tablet/TabletBase.java | 3 +-
.../apache/accumulo/tserver/tablet/TabletData.java | 97 ----
.../accumulo/test/performance/NullTserver.java | 3 +-
21 files changed, 511 insertions(+), 1493 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index a0a567212c..41dfbbcdfb 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -643,8 +643,6 @@ public enum Property {
+ " minor compacted file because it may have been modified by
iterators. The"
+ " file dumped to the local dir is an exact copy of what was in
memory.",
"1.3.5"),
- TSERV_HEALTH_CHECK_FREQ("tserver.health.check.interval", "30m",
PropertyType.TIMEDURATION,
- "The time between tablet server health checks.", "2.1.0"),
TSERV_MINTHREADS("tserver.server.threads.minimum", "20", PropertyType.COUNT,
"The minimum number of threads to use to handle incoming requests.",
"1.4.0"),
TSERV_MINTHREADS_TIMEOUT("tserver.server.threads.timeout", "0s",
PropertyType.TIMEDURATION,
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
index ae36e63832..e737805761 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
@@ -158,14 +158,14 @@ public class TabletLogger {
fileLog.debug("Imported {} {} ", extent, file);
}
- public static void recovering(KeyExtent extent, List<LogEntry> logEntries) {
+ public static void recovering(KeyExtent extent, Collection<LogEntry>
logEntries) {
if (recoveryLog.isDebugEnabled()) {
List<String> logIds =
logEntries.stream().map(LogEntry::getUniqueID).collect(toList());
recoveryLog.debug("For {} recovering data from walogs: {}", extent,
logIds);
}
}
- public static void recovered(KeyExtent extent, List<LogEntry> logEntries,
long numMutation,
+ public static void recovered(KeyExtent extent, Collection<LogEntry>
logEntries, long numMutation,
long numEntries) {
recoveryLog.info("For {} recovered {} mutations creating {} entries from
{} walogs", extent,
numMutation, numEntries, logEntries.size());
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index 6a9dd94d97..d129d04164 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@ -124,5 +124,4 @@ public class StoredTabletFile extends
AbstractTabletFile<StoredTabletFile> {
public static StoredTabletFile of(final String metadataEntry) {
return new StoredTabletFile(metadataEntry);
}
-
}
diff --git
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TTabletRefresh.java
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TTabletRefresh.java
deleted file mode 100644
index eb59bdc17f..0000000000
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TTabletRefresh.java
+++ /dev/null
@@ -1,566 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Autogenerated by Thrift Compiler (0.17.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.accumulo.core.tabletserver.thrift;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"})
-public class TTabletRefresh implements org.apache.thrift.TBase<TTabletRefresh,
TTabletRefresh._Fields>, java.io.Serializable, Cloneable,
Comparable<TTabletRefresh> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new
org.apache.thrift.protocol.TStruct("TTabletRefresh");
-
- private static final org.apache.thrift.protocol.TField EXTENT_FIELD_DESC =
new org.apache.thrift.protocol.TField("extent",
org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField
SCAN_ENTRIES_FIELD_DESC = new org.apache.thrift.protocol.TField("scanEntries",
org.apache.thrift.protocol.TType.LIST, (short)2);
-
- private static final org.apache.thrift.scheme.SchemeFactory
STANDARD_SCHEME_FACTORY = new TTabletRefreshStandardSchemeFactory();
- private static final org.apache.thrift.scheme.SchemeFactory
TUPLE_SCHEME_FACTORY = new TTabletRefreshTupleSchemeFactory();
-
- public @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; // required
- public @org.apache.thrift.annotation.Nullable
java.util.List<java.lang.String> scanEntries; // required
-
- /** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- EXTENT((short)1, "extent"),
- SCAN_ENTRIES((short)2, "scanEntries");
-
- private static final java.util.Map<java.lang.String, _Fields> byName = new
java.util.HashMap<java.lang.String, _Fields>();
-
- static {
- for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not
found.
- */
- @org.apache.thrift.annotation.Nullable
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // EXTENT
- return EXTENT;
- case 2: // SCAN_ENTRIES
- return SCAN_ENTRIES;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new java.lang.IllegalArgumentException("Field
" + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- @org.apache.thrift.annotation.Nullable
- public static _Fields findByName(java.lang.String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final java.lang.String _fieldName;
-
- _Fields(short thriftId, java.lang.String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- @Override
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- @Override
- public java.lang.String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final java.util.Map<_Fields,
org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap =
new java.util.EnumMap<_Fields,
org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.EXTENT, new
org.apache.thrift.meta_data.FieldMetaData("extent",
org.apache.thrift.TFieldRequirementType.DEFAULT,
- new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class)));
- tmpMap.put(_Fields.SCAN_ENTRIES, new
org.apache.thrift.meta_data.FieldMetaData("scanEntries",
org.apache.thrift.TFieldRequirementType.DEFAULT,
- new
org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
-
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTabletRefresh.class,
metaDataMap);
- }
-
- public TTabletRefresh() {
- }
-
- public TTabletRefresh(
- org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent,
- java.util.List<java.lang.String> scanEntries)
- {
- this();
- this.extent = extent;
- this.scanEntries = scanEntries;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TTabletRefresh(TTabletRefresh other) {
- if (other.isSetExtent()) {
- this.extent = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(other.extent);
- }
- if (other.isSetScanEntries()) {
- java.util.List<java.lang.String> __this__scanEntries = new
java.util.ArrayList<java.lang.String>(other.scanEntries);
- this.scanEntries = __this__scanEntries;
- }
- }
-
- @Override
- public TTabletRefresh deepCopy() {
- return new TTabletRefresh(this);
- }
-
- @Override
- public void clear() {
- this.extent = null;
- this.scanEntries = null;
- }
-
- @org.apache.thrift.annotation.Nullable
- public org.apache.accumulo.core.dataImpl.thrift.TKeyExtent getExtent() {
- return this.extent;
- }
-
- public TTabletRefresh setExtent(@org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) {
- this.extent = extent;
- return this;
- }
-
- public void unsetExtent() {
- this.extent = null;
- }
-
- /** Returns true if field extent is set (has been assigned a value) and
false otherwise */
- public boolean isSetExtent() {
- return this.extent != null;
- }
-
- public void setExtentIsSet(boolean value) {
- if (!value) {
- this.extent = null;
- }
- }
-
- public int getScanEntriesSize() {
- return (this.scanEntries == null) ? 0 : this.scanEntries.size();
- }
-
- @org.apache.thrift.annotation.Nullable
- public java.util.Iterator<java.lang.String> getScanEntriesIterator() {
- return (this.scanEntries == null) ? null : this.scanEntries.iterator();
- }
-
- public void addToScanEntries(java.lang.String elem) {
- if (this.scanEntries == null) {
- this.scanEntries = new java.util.ArrayList<java.lang.String>();
- }
- this.scanEntries.add(elem);
- }
-
- @org.apache.thrift.annotation.Nullable
- public java.util.List<java.lang.String> getScanEntries() {
- return this.scanEntries;
- }
-
- public TTabletRefresh setScanEntries(@org.apache.thrift.annotation.Nullable
java.util.List<java.lang.String> scanEntries) {
- this.scanEntries = scanEntries;
- return this;
- }
-
- public void unsetScanEntries() {
- this.scanEntries = null;
- }
-
- /** Returns true if field scanEntries is set (has been assigned a value) and
false otherwise */
- public boolean isSetScanEntries() {
- return this.scanEntries != null;
- }
-
- public void setScanEntriesIsSet(boolean value) {
- if (!value) {
- this.scanEntries = null;
- }
- }
-
- @Override
- public void setFieldValue(_Fields field,
@org.apache.thrift.annotation.Nullable java.lang.Object value) {
- switch (field) {
- case EXTENT:
- if (value == null) {
- unsetExtent();
- } else {
- setExtent((org.apache.accumulo.core.dataImpl.thrift.TKeyExtent)value);
- }
- break;
-
- case SCAN_ENTRIES:
- if (value == null) {
- unsetScanEntries();
- } else {
- setScanEntries((java.util.List<java.lang.String>)value);
- }
- break;
-
- }
- }
-
- @org.apache.thrift.annotation.Nullable
- @Override
- public java.lang.Object getFieldValue(_Fields field) {
- switch (field) {
- case EXTENT:
- return getExtent();
-
- case SCAN_ENTRIES:
- return getScanEntries();
-
- }
- throw new java.lang.IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned
a value) and false otherwise */
- @Override
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new java.lang.IllegalArgumentException();
- }
-
- switch (field) {
- case EXTENT:
- return isSetExtent();
- case SCAN_ENTRIES:
- return isSetScanEntries();
- }
- throw new java.lang.IllegalStateException();
- }
-
- @Override
- public boolean equals(java.lang.Object that) {
- if (that instanceof TTabletRefresh)
- return this.equals((TTabletRefresh)that);
- return false;
- }
-
- public boolean equals(TTabletRefresh that) {
- if (that == null)
- return false;
- if (this == that)
- return true;
-
- boolean this_present_extent = true && this.isSetExtent();
- boolean that_present_extent = true && that.isSetExtent();
- if (this_present_extent || that_present_extent) {
- if (!(this_present_extent && that_present_extent))
- return false;
- if (!this.extent.equals(that.extent))
- return false;
- }
-
- boolean this_present_scanEntries = true && this.isSetScanEntries();
- boolean that_present_scanEntries = true && that.isSetScanEntries();
- if (this_present_scanEntries || that_present_scanEntries) {
- if (!(this_present_scanEntries && that_present_scanEntries))
- return false;
- if (!this.scanEntries.equals(that.scanEntries))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int hashCode = 1;
-
- hashCode = hashCode * 8191 + ((isSetExtent()) ? 131071 : 524287);
- if (isSetExtent())
- hashCode = hashCode * 8191 + extent.hashCode();
-
- hashCode = hashCode * 8191 + ((isSetScanEntries()) ? 131071 : 524287);
- if (isSetScanEntries())
- hashCode = hashCode * 8191 + scanEntries.hashCode();
-
- return hashCode;
- }
-
- @Override
- public int compareTo(TTabletRefresh other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = java.lang.Boolean.compare(isSetExtent(),
other.isSetExtent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetExtent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.extent,
other.extent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = java.lang.Boolean.compare(isSetScanEntries(),
other.isSetScanEntries());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetScanEntries()) {
- lastComparison =
org.apache.thrift.TBaseHelper.compareTo(this.scanEntries, other.scanEntries);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- @org.apache.thrift.annotation.Nullable
- @Override
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws
org.apache.thrift.TException {
- scheme(iprot).read(iprot, this);
- }
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws
org.apache.thrift.TException {
- scheme(oprot).write(oprot, this);
- }
-
- @Override
- public java.lang.String toString() {
- java.lang.StringBuilder sb = new
java.lang.StringBuilder("TTabletRefresh(");
- boolean first = true;
-
- sb.append("extent:");
- if (this.extent == null) {
- sb.append("null");
- } else {
- sb.append(this.extent);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("scanEntries:");
- if (this.scanEntries == null) {
- sb.append("null");
- } else {
- sb.append(this.scanEntries);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (extent != null) {
- extent.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws
java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new
org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws
java.io.IOException, java.lang.ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new
org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TTabletRefreshStandardSchemeFactory implements
org.apache.thrift.scheme.SchemeFactory {
- @Override
- public TTabletRefreshStandardScheme getScheme() {
- return new TTabletRefreshStandardScheme();
- }
- }
-
- private static class TTabletRefreshStandardScheme extends
org.apache.thrift.scheme.StandardScheme<TTabletRefresh> {
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol iprot,
TTabletRefresh struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // EXTENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.extent = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
- struct.extent.read(iprot);
- struct.setExtentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- break;
- case 2: // SCAN_ENTRIES
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list72 =
iprot.readListBegin();
- struct.scanEntries = new
java.util.ArrayList<java.lang.String>(_list72.size);
- @org.apache.thrift.annotation.Nullable java.lang.String
_elem73;
- for (int _i74 = 0; _i74 < _list72.size; ++_i74)
- {
- _elem73 = iprot.readString();
- struct.scanEntries.add(_elem73);
- }
- iprot.readListEnd();
- }
- struct.setScanEntriesIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked
in the validate method
- struct.validate();
- }
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol oprot,
TTabletRefresh struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.extent != null) {
- oprot.writeFieldBegin(EXTENT_FIELD_DESC);
- struct.extent.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.scanEntries != null) {
- oprot.writeFieldBegin(SCAN_ENTRIES_FIELD_DESC);
- {
- oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
struct.scanEntries.size()));
- for (java.lang.String _iter75 : struct.scanEntries)
- {
- oprot.writeString(_iter75);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TTabletRefreshTupleSchemeFactory implements
org.apache.thrift.scheme.SchemeFactory {
- @Override
- public TTabletRefreshTupleScheme getScheme() {
- return new TTabletRefreshTupleScheme();
- }
- }
-
- private static class TTabletRefreshTupleScheme extends
org.apache.thrift.scheme.TupleScheme<TTabletRefresh> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot,
TTabletRefresh struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TTupleProtocol oprot =
(org.apache.thrift.protocol.TTupleProtocol) prot;
- java.util.BitSet optionals = new java.util.BitSet();
- if (struct.isSetExtent()) {
- optionals.set(0);
- }
- if (struct.isSetScanEntries()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetExtent()) {
- struct.extent.write(oprot);
- }
- if (struct.isSetScanEntries()) {
- {
- oprot.writeI32(struct.scanEntries.size());
- for (java.lang.String _iter76 : struct.scanEntries)
- {
- oprot.writeString(_iter76);
- }
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TTabletRefresh
struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TTupleProtocol iprot =
(org.apache.thrift.protocol.TTupleProtocol) prot;
- java.util.BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.extent = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
- struct.extent.read(iprot);
- struct.setExtentIsSet(true);
- }
- if (incoming.get(1)) {
- {
- org.apache.thrift.protocol.TList _list77 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRING);
- struct.scanEntries = new
java.util.ArrayList<java.lang.String>(_list77.size);
- @org.apache.thrift.annotation.Nullable java.lang.String _elem78;
- for (int _i79 = 0; _i79 < _list77.size; ++_i79)
- {
- _elem78 = iprot.readString();
- struct.scanEntries.add(_elem78);
- }
- }
- struct.setScanEntriesIsSet(true);
- }
- }
- }
-
- private static <S extends org.apache.thrift.scheme.IScheme> S
scheme(org.apache.thrift.protocol.TProtocol proto) {
- return
(org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ?
STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme();
- }
- private static void unusedMethod() {}
-}
-
diff --git
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TabletServerClientService.java
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TabletServerClientService.java
index b36acd1424..b9cb50df14 100644
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TabletServerClientService.java
+++
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TabletServerClientService.java
@@ -53,7 +53,7 @@ public class TabletServerClientService {
public org.apache.accumulo.core.dataImpl.thrift.TSummaries
contiuneGetSummaries(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
long sessionId) throws NoSuchScanIDException, org.apache.thrift.TException;
- public java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<TTabletRefresh> tabletsToRefresh) throws
org.apache.thrift.TException;
+ public java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh) throws org.apache.thrift.TException;
}
@@ -83,7 +83,7 @@ public class TabletServerClientService {
public void
contiuneGetSummaries(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
long sessionId,
org.apache.thrift.async.AsyncMethodCallback<org.apache.accumulo.core.dataImpl.thrift.TSummaries>
resultHandler) throws org.apache.thrift.TException;
- public void
refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<TTabletRefresh> tabletsToRefresh,
org.apache.thrift.async.AsyncMethodCallback<java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>>
resultHandler) throws org.apache.thrift.TException;
+ public void
refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh,
org.apache.thrift.async.AsyncMethodCallback<java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>>
resultHandler) throws org.apache.thrift.TException;
}
@@ -415,13 +415,13 @@ public class TabletServerClientService {
}
@Override
- public java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<TTabletRefresh> tabletsToRefresh) throws
org.apache.thrift.TException
+ public java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh) throws org.apache.thrift.TException
{
send_refreshTablets(tinfo, credentials, tabletsToRefresh);
return recv_refreshTablets();
}
- public void
send_refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<TTabletRefresh> tabletsToRefresh) throws
org.apache.thrift.TException
+ public void
send_refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh) throws org.apache.thrift.TException
{
refreshTablets_args args = new refreshTablets_args();
args.setTinfo(tinfo);
@@ -959,7 +959,7 @@ public class TabletServerClientService {
}
@Override
- public void
refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<TTabletRefresh> tabletsToRefresh,
org.apache.thrift.async.AsyncMethodCallback<java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>>
resultHandler) throws org.apache.thrift.TException {
+ public void
refreshTablets(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh,
org.apache.thrift.async.AsyncMethodCallback<java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>>
resultHandler) throws org.apache.thrift.TException {
checkReady();
refreshTablets_call method_call = new refreshTablets_call(tinfo,
credentials, tabletsToRefresh, resultHandler, this, ___protocolFactory,
___transport);
this.___currentMethod = method_call;
@@ -969,8 +969,8 @@ public class TabletServerClientService {
public static class refreshTablets_call extends
org.apache.thrift.async.TAsyncMethodCall<java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>>
{
private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo;
private org.apache.accumulo.core.securityImpl.thrift.TCredentials
credentials;
- private java.util.List<TTabletRefresh> tabletsToRefresh;
- public
refreshTablets_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<TTabletRefresh> tabletsToRefresh,
org.apache.thrift.async.AsyncMethodCallback<java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>>
resultHandler, org.apache.thrift.async.TAsyncClient client,
org.apache.thrift.protocol.TProtocolFactory protocolFactory,
org.apache.thrift.transport.TNonblockingTransport [...]
+ private
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh;
+ public
refreshTablets_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh,
org.apache.thrift.async.AsyncMethodCallback<java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>>
resultHandler, org.apache.thrift.async.TAsyncClient client,
org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.t [...]
super(client, protocolFactory, transport, resultHandler, false);
this.tinfo = tinfo;
this.credentials = credentials;
@@ -5213,14 +5213,14 @@ public class TabletServerClientService {
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list80 =
iprot.readListBegin();
- struct.success = new
java.util.ArrayList<TabletStats>(_list80.size);
- @org.apache.thrift.annotation.Nullable TabletStats _elem81;
- for (int _i82 = 0; _i82 < _list80.size; ++_i82)
+ org.apache.thrift.protocol.TList _list72 =
iprot.readListBegin();
+ struct.success = new
java.util.ArrayList<TabletStats>(_list72.size);
+ @org.apache.thrift.annotation.Nullable TabletStats _elem73;
+ for (int _i74 = 0; _i74 < _list72.size; ++_i74)
{
- _elem81 = new TabletStats();
- _elem81.read(iprot);
- struct.success.add(_elem81);
+ _elem73 = new TabletStats();
+ _elem73.read(iprot);
+ struct.success.add(_elem73);
}
iprot.readListEnd();
}
@@ -5258,9 +5258,9 @@ public class TabletServerClientService {
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.success.size()));
- for (TabletStats _iter83 : struct.success)
+ for (TabletStats _iter75 : struct.success)
{
- _iter83.write(oprot);
+ _iter75.write(oprot);
}
oprot.writeListEnd();
}
@@ -5300,9 +5300,9 @@ public class TabletServerClientService {
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
- for (TabletStats _iter84 : struct.success)
+ for (TabletStats _iter76 : struct.success)
{
- _iter84.write(oprot);
+ _iter76.write(oprot);
}
}
}
@@ -5317,14 +5317,14 @@ public class TabletServerClientService {
java.util.BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list85 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
- struct.success = new
java.util.ArrayList<TabletStats>(_list85.size);
- @org.apache.thrift.annotation.Nullable TabletStats _elem86;
- for (int _i87 = 0; _i87 < _list85.size; ++_i87)
+ org.apache.thrift.protocol.TList _list77 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
+ struct.success = new
java.util.ArrayList<TabletStats>(_list77.size);
+ @org.apache.thrift.annotation.Nullable TabletStats _elem78;
+ for (int _i79 = 0; _i79 < _list77.size; ++_i79)
{
- _elem86 = new TabletStats();
- _elem86.read(iprot);
- struct.success.add(_elem86);
+ _elem78 = new TabletStats();
+ _elem78.read(iprot);
+ struct.success.add(_elem78);
}
}
struct.setSuccessIsSet(true);
@@ -8419,13 +8419,13 @@ public class TabletServerClientService {
case 3: // FILENAMES
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list88 =
iprot.readListBegin();
- struct.filenames = new
java.util.ArrayList<java.lang.String>(_list88.size);
- @org.apache.thrift.annotation.Nullable java.lang.String
_elem89;
- for (int _i90 = 0; _i90 < _list88.size; ++_i90)
+ org.apache.thrift.protocol.TList _list80 =
iprot.readListBegin();
+ struct.filenames = new
java.util.ArrayList<java.lang.String>(_list80.size);
+ @org.apache.thrift.annotation.Nullable java.lang.String
_elem81;
+ for (int _i82 = 0; _i82 < _list80.size; ++_i82)
{
- _elem89 = iprot.readString();
- struct.filenames.add(_elem89);
+ _elem81 = iprot.readString();
+ struct.filenames.add(_elem81);
}
iprot.readListEnd();
}
@@ -8464,9 +8464,9 @@ public class TabletServerClientService {
oprot.writeFieldBegin(FILENAMES_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
struct.filenames.size()));
- for (java.lang.String _iter91 : struct.filenames)
+ for (java.lang.String _iter83 : struct.filenames)
{
- oprot.writeString(_iter91);
+ oprot.writeString(_iter83);
}
oprot.writeListEnd();
}
@@ -8510,9 +8510,9 @@ public class TabletServerClientService {
if (struct.isSetFilenames()) {
{
oprot.writeI32(struct.filenames.size());
- for (java.lang.String _iter92 : struct.filenames)
+ for (java.lang.String _iter84 : struct.filenames)
{
- oprot.writeString(_iter92);
+ oprot.writeString(_iter84);
}
}
}
@@ -8534,13 +8534,13 @@ public class TabletServerClientService {
}
if (incoming.get(2)) {
{
- org.apache.thrift.protocol.TList _list93 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRING);
- struct.filenames = new
java.util.ArrayList<java.lang.String>(_list93.size);
- @org.apache.thrift.annotation.Nullable java.lang.String _elem94;
- for (int _i95 = 0; _i95 < _list93.size; ++_i95)
+ org.apache.thrift.protocol.TList _list85 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRING);
+ struct.filenames = new
java.util.ArrayList<java.lang.String>(_list85.size);
+ @org.apache.thrift.annotation.Nullable java.lang.String _elem86;
+ for (int _i87 = 0; _i87 < _list85.size; ++_i87)
{
- _elem94 = iprot.readString();
- struct.filenames.add(_elem94);
+ _elem86 = iprot.readString();
+ struct.filenames.add(_elem86);
}
}
struct.setFilenamesIsSet(true);
@@ -9373,13 +9373,13 @@ public class TabletServerClientService {
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list96 =
iprot.readListBegin();
- struct.success = new
java.util.ArrayList<java.lang.String>(_list96.size);
- @org.apache.thrift.annotation.Nullable java.lang.String
_elem97;
- for (int _i98 = 0; _i98 < _list96.size; ++_i98)
+ org.apache.thrift.protocol.TList _list88 =
iprot.readListBegin();
+ struct.success = new
java.util.ArrayList<java.lang.String>(_list88.size);
+ @org.apache.thrift.annotation.Nullable java.lang.String
_elem89;
+ for (int _i90 = 0; _i90 < _list88.size; ++_i90)
{
- _elem97 = iprot.readString();
- struct.success.add(_elem97);
+ _elem89 = iprot.readString();
+ struct.success.add(_elem89);
}
iprot.readListEnd();
}
@@ -9408,9 +9408,9 @@ public class TabletServerClientService {
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
struct.success.size()));
- for (java.lang.String _iter99 : struct.success)
+ for (java.lang.String _iter91 : struct.success)
{
- oprot.writeString(_iter99);
+ oprot.writeString(_iter91);
}
oprot.writeListEnd();
}
@@ -9442,9 +9442,9 @@ public class TabletServerClientService {
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
- for (java.lang.String _iter100 : struct.success)
+ for (java.lang.String _iter92 : struct.success)
{
- oprot.writeString(_iter100);
+ oprot.writeString(_iter92);
}
}
}
@@ -9456,13 +9456,13 @@ public class TabletServerClientService {
java.util.BitSet incoming = iprot.readBitSet(1);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list101 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRING);
- struct.success = new
java.util.ArrayList<java.lang.String>(_list101.size);
- @org.apache.thrift.annotation.Nullable java.lang.String _elem102;
- for (int _i103 = 0; _i103 < _list101.size; ++_i103)
+ org.apache.thrift.protocol.TList _list93 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRING);
+ struct.success = new
java.util.ArrayList<java.lang.String>(_list93.size);
+ @org.apache.thrift.annotation.Nullable java.lang.String _elem94;
+ for (int _i95 = 0; _i95 < _list93.size; ++_i95)
{
- _elem102 = iprot.readString();
- struct.success.add(_elem102);
+ _elem94 = iprot.readString();
+ struct.success.add(_elem94);
}
}
struct.setSuccessIsSet(true);
@@ -12586,26 +12586,26 @@ public class TabletServerClientService {
case 4: // FILES
if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
{
- org.apache.thrift.protocol.TMap _map104 =
iprot.readMapBegin();
- struct.files = new
java.util.HashMap<java.lang.String,java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>>(2*_map104.size);
- @org.apache.thrift.annotation.Nullable java.lang.String
_key105;
- @org.apache.thrift.annotation.Nullable
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange> _val106;
- for (int _i107 = 0; _i107 < _map104.size; ++_i107)
+ org.apache.thrift.protocol.TMap _map96 =
iprot.readMapBegin();
+ struct.files = new
java.util.HashMap<java.lang.String,java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>>(2*_map96.size);
+ @org.apache.thrift.annotation.Nullable java.lang.String
_key97;
+ @org.apache.thrift.annotation.Nullable
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange> _val98;
+ for (int _i99 = 0; _i99 < _map96.size; ++_i99)
{
- _key105 = iprot.readString();
+ _key97 = iprot.readString();
{
- org.apache.thrift.protocol.TList _list108 =
iprot.readListBegin();
- _val106 = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TRowRange>(_list108.size);
- @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TRowRange _elem109;
- for (int _i110 = 0; _i110 < _list108.size; ++_i110)
+ org.apache.thrift.protocol.TList _list100 =
iprot.readListBegin();
+ _val98 = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TRowRange>(_list100.size);
+ @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TRowRange _elem101;
+ for (int _i102 = 0; _i102 < _list100.size; ++_i102)
{
- _elem109 = new
org.apache.accumulo.core.dataImpl.thrift.TRowRange();
- _elem109.read(iprot);
- _val106.add(_elem109);
+ _elem101 = new
org.apache.accumulo.core.dataImpl.thrift.TRowRange();
+ _elem101.read(iprot);
+ _val98.add(_elem101);
}
iprot.readListEnd();
}
- struct.files.put(_key105, _val106);
+ struct.files.put(_key97, _val98);
}
iprot.readMapEnd();
}
@@ -12649,14 +12649,14 @@ public class TabletServerClientService {
oprot.writeFieldBegin(FILES_FIELD_DESC);
{
oprot.writeMapBegin(new
org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.LIST, struct.files.size()));
- for (java.util.Map.Entry<java.lang.String,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>> _iter111 :
struct.files.entrySet())
+ for (java.util.Map.Entry<java.lang.String,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>> _iter103 :
struct.files.entrySet())
{
- oprot.writeString(_iter111.getKey());
+ oprot.writeString(_iter103.getKey());
{
- oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
_iter111.getValue().size()));
- for (org.apache.accumulo.core.dataImpl.thrift.TRowRange
_iter112 : _iter111.getValue())
+ oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
_iter103.getValue().size()));
+ for (org.apache.accumulo.core.dataImpl.thrift.TRowRange
_iter104 : _iter103.getValue())
{
- _iter112.write(oprot);
+ _iter104.write(oprot);
}
oprot.writeListEnd();
}
@@ -12709,14 +12709,14 @@ public class TabletServerClientService {
if (struct.isSetFiles()) {
{
oprot.writeI32(struct.files.size());
- for (java.util.Map.Entry<java.lang.String,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>> _iter113 :
struct.files.entrySet())
+ for (java.util.Map.Entry<java.lang.String,
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>> _iter105 :
struct.files.entrySet())
{
- oprot.writeString(_iter113.getKey());
+ oprot.writeString(_iter105.getKey());
{
- oprot.writeI32(_iter113.getValue().size());
- for (org.apache.accumulo.core.dataImpl.thrift.TRowRange
_iter114 : _iter113.getValue())
+ oprot.writeI32(_iter105.getValue().size());
+ for (org.apache.accumulo.core.dataImpl.thrift.TRowRange
_iter106 : _iter105.getValue())
{
- _iter114.write(oprot);
+ _iter106.write(oprot);
}
}
}
@@ -12745,25 +12745,25 @@ public class TabletServerClientService {
}
if (incoming.get(3)) {
{
- org.apache.thrift.protocol.TMap _map115 =
iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.LIST);
- struct.files = new
java.util.HashMap<java.lang.String,java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>>(2*_map115.size);
- @org.apache.thrift.annotation.Nullable java.lang.String _key116;
- @org.apache.thrift.annotation.Nullable
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange> _val117;
- for (int _i118 = 0; _i118 < _map115.size; ++_i118)
+ org.apache.thrift.protocol.TMap _map107 =
iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.LIST);
+ struct.files = new
java.util.HashMap<java.lang.String,java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>>(2*_map107.size);
+ @org.apache.thrift.annotation.Nullable java.lang.String _key108;
+ @org.apache.thrift.annotation.Nullable
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange> _val109;
+ for (int _i110 = 0; _i110 < _map107.size; ++_i110)
{
- _key116 = iprot.readString();
+ _key108 = iprot.readString();
{
- org.apache.thrift.protocol.TList _list119 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
- _val117 = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TRowRange>(_list119.size);
- @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TRowRange _elem120;
- for (int _i121 = 0; _i121 < _list119.size; ++_i121)
+ org.apache.thrift.protocol.TList _list111 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
+ _val109 = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TRowRange>(_list111.size);
+ @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TRowRange _elem112;
+ for (int _i113 = 0; _i113 < _list111.size; ++_i113)
{
- _elem120 = new
org.apache.accumulo.core.dataImpl.thrift.TRowRange();
- _elem120.read(iprot);
- _val117.add(_elem120);
+ _elem112 = new
org.apache.accumulo.core.dataImpl.thrift.TRowRange();
+ _elem112.read(iprot);
+ _val109.add(_elem112);
}
}
- struct.files.put(_key116, _val117);
+ struct.files.put(_key108, _val109);
}
}
struct.setFilesIsSet(true);
@@ -14253,7 +14253,7 @@ public class TabletServerClientService {
public @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required
public @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; //
required
- public @org.apache.thrift.annotation.Nullable
java.util.List<TTabletRefresh> tabletsToRefresh; // required
+ public @org.apache.thrift.annotation.Nullable
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -14333,7 +14333,7 @@ public class TabletServerClientService {
new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
org.apache.accumulo.core.securityImpl.thrift.TCredentials.class)));
tmpMap.put(_Fields.TABLETS_TO_REFRESH, new
org.apache.thrift.meta_data.FieldMetaData("tabletsToRefresh",
org.apache.thrift.TFieldRequirementType.DEFAULT,
new
org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
TTabletRefresh.class))));
+ new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class))));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(refreshTablets_args.class,
metaDataMap);
}
@@ -14344,7 +14344,7 @@ public class TabletServerClientService {
public refreshTablets_args(
org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
- java.util.List<TTabletRefresh> tabletsToRefresh)
+ java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh)
{
this();
this.tinfo = tinfo;
@@ -14363,9 +14363,9 @@ public class TabletServerClientService {
this.credentials = new
org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials);
}
if (other.isSetTabletsToRefresh()) {
- java.util.List<TTabletRefresh> __this__tabletsToRefresh = new
java.util.ArrayList<TTabletRefresh>(other.tabletsToRefresh.size());
- for (TTabletRefresh other_element : other.tabletsToRefresh) {
- __this__tabletsToRefresh.add(new TTabletRefresh(other_element));
+ java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
__this__tabletsToRefresh = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>(other.tabletsToRefresh.size());
+ for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent other_element
: other.tabletsToRefresh) {
+ __this__tabletsToRefresh.add(new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(other_element));
}
this.tabletsToRefresh = __this__tabletsToRefresh;
}
@@ -14438,23 +14438,23 @@ public class TabletServerClientService {
}
@org.apache.thrift.annotation.Nullable
- public java.util.Iterator<TTabletRefresh> getTabletsToRefreshIterator() {
+ public
java.util.Iterator<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
getTabletsToRefreshIterator() {
return (this.tabletsToRefresh == null) ? null :
this.tabletsToRefresh.iterator();
}
- public void addToTabletsToRefresh(TTabletRefresh elem) {
+ public void
addToTabletsToRefresh(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent elem)
{
if (this.tabletsToRefresh == null) {
- this.tabletsToRefresh = new java.util.ArrayList<TTabletRefresh>();
+ this.tabletsToRefresh = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>();
}
this.tabletsToRefresh.add(elem);
}
@org.apache.thrift.annotation.Nullable
- public java.util.List<TTabletRefresh> getTabletsToRefresh() {
+ public java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
getTabletsToRefresh() {
return this.tabletsToRefresh;
}
- public refreshTablets_args
setTabletsToRefresh(@org.apache.thrift.annotation.Nullable
java.util.List<TTabletRefresh> tabletsToRefresh) {
+ public refreshTablets_args
setTabletsToRefresh(@org.apache.thrift.annotation.Nullable
java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>
tabletsToRefresh) {
this.tabletsToRefresh = tabletsToRefresh;
return this;
}
@@ -14497,7 +14497,7 @@ public class TabletServerClientService {
if (value == null) {
unsetTabletsToRefresh();
} else {
- setTabletsToRefresh((java.util.List<TTabletRefresh>)value);
+
setTabletsToRefresh((java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>)value);
}
break;
@@ -14758,14 +14758,14 @@ public class TabletServerClientService {
case 3: // TABLETS_TO_REFRESH
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list122 =
iprot.readListBegin();
- struct.tabletsToRefresh = new
java.util.ArrayList<TTabletRefresh>(_list122.size);
- @org.apache.thrift.annotation.Nullable TTabletRefresh
_elem123;
- for (int _i124 = 0; _i124 < _list122.size; ++_i124)
+ org.apache.thrift.protocol.TList _list114 =
iprot.readListBegin();
+ struct.tabletsToRefresh = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>(_list114.size);
+ @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem115;
+ for (int _i116 = 0; _i116 < _list114.size; ++_i116)
{
- _elem123 = new TTabletRefresh();
- _elem123.read(iprot);
- struct.tabletsToRefresh.add(_elem123);
+ _elem115 = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
+ _elem115.read(iprot);
+ struct.tabletsToRefresh.add(_elem115);
}
iprot.readListEnd();
}
@@ -14804,9 +14804,9 @@ public class TabletServerClientService {
oprot.writeFieldBegin(TABLETS_TO_REFRESH_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.tabletsToRefresh.size()));
- for (TTabletRefresh _iter125 : struct.tabletsToRefresh)
+ for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter117
: struct.tabletsToRefresh)
{
- _iter125.write(oprot);
+ _iter117.write(oprot);
}
oprot.writeListEnd();
}
@@ -14850,9 +14850,9 @@ public class TabletServerClientService {
if (struct.isSetTabletsToRefresh()) {
{
oprot.writeI32(struct.tabletsToRefresh.size());
- for (TTabletRefresh _iter126 : struct.tabletsToRefresh)
+ for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter118
: struct.tabletsToRefresh)
{
- _iter126.write(oprot);
+ _iter118.write(oprot);
}
}
}
@@ -14874,14 +14874,14 @@ public class TabletServerClientService {
}
if (incoming.get(2)) {
{
- org.apache.thrift.protocol.TList _list127 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
- struct.tabletsToRefresh = new
java.util.ArrayList<TTabletRefresh>(_list127.size);
- @org.apache.thrift.annotation.Nullable TTabletRefresh _elem128;
- for (int _i129 = 0; _i129 < _list127.size; ++_i129)
+ org.apache.thrift.protocol.TList _list119 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
+ struct.tabletsToRefresh = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>(_list119.size);
+ @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem120;
+ for (int _i121 = 0; _i121 < _list119.size; ++_i121)
{
- _elem128 = new TTabletRefresh();
- _elem128.read(iprot);
- struct.tabletsToRefresh.add(_elem128);
+ _elem120 = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
+ _elem120.read(iprot);
+ struct.tabletsToRefresh.add(_elem120);
}
}
struct.setTabletsToRefreshIsSet(true);
@@ -15223,14 +15223,14 @@ public class TabletServerClientService {
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list130 =
iprot.readListBegin();
- struct.success = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>(_list130.size);
- @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem131;
- for (int _i132 = 0; _i132 < _list130.size; ++_i132)
+ org.apache.thrift.protocol.TList _list122 =
iprot.readListBegin();
+ struct.success = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>(_list122.size);
+ @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem123;
+ for (int _i124 = 0; _i124 < _list122.size; ++_i124)
{
- _elem131 = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
- _elem131.read(iprot);
- struct.success.add(_elem131);
+ _elem123 = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
+ _elem123.read(iprot);
+ struct.success.add(_elem123);
}
iprot.readListEnd();
}
@@ -15259,9 +15259,9 @@ public class TabletServerClientService {
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.success.size()));
- for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter133
: struct.success)
+ for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter125
: struct.success)
{
- _iter133.write(oprot);
+ _iter125.write(oprot);
}
oprot.writeListEnd();
}
@@ -15293,9 +15293,9 @@ public class TabletServerClientService {
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
- for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter134
: struct.success)
+ for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter126
: struct.success)
{
- _iter134.write(oprot);
+ _iter126.write(oprot);
}
}
}
@@ -15307,14 +15307,14 @@ public class TabletServerClientService {
java.util.BitSet incoming = iprot.readBitSet(1);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list135 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
- struct.success = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>(_list135.size);
- @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem136;
- for (int _i137 = 0; _i137 < _list135.size; ++_i137)
+ org.apache.thrift.protocol.TList _list127 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
+ struct.success = new
java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>(_list127.size);
+ @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem128;
+ for (int _i129 = 0; _i129 < _list127.size; ++_i129)
{
- _elem136 = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
- _elem136.read(iprot);
- struct.success.add(_elem136);
+ _elem128 = new
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
+ _elem128.read(iprot);
+ struct.success.add(_elem128);
}
}
struct.setSuccessIsSet(true);
diff --git a/core/src/main/thrift/tabletserver.thrift
b/core/src/main/thrift/tabletserver.thrift
index b642e307f0..535b38b280 100644
--- a/core/src/main/thrift/tabletserver.thrift
+++ b/core/src/main/thrift/tabletserver.thrift
@@ -134,11 +134,6 @@ struct TCompactionStats{
3:i64 fileSize;
}
-struct TTabletRefresh {
- 1:data.TKeyExtent extent
- 2:list<string> scanEntries
-}
-
service TabletServerClientService {
oneway void flush(
@@ -235,7 +230,7 @@ service TabletServerClientService {
list<data.TKeyExtent> refreshTablets(
1:client.TInfo tinfo
2:security.TCredentials credentials
- 3:list<TTabletRefresh> tabletsToRefresh
+ 3:list<data.TKeyExtent> tabletsToRefresh
)
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 2dbdbf8568..76eeaf892e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -127,6 +127,8 @@ public class VolumeUtil {
}
}
+ // ELASTICITY_TODO this method is no longer called because volume
replacement needs to move from
+ // the tablet server to the manager. See #3625
/**
* This method does two things. First, it switches any volumes a tablet is
using that are
* configured in instance.volumes.replacements. Second, if a tablet dir is
no longer configured
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 5a518083be..0ce5bbe0ab 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -30,6 +30,7 @@ import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -411,7 +412,7 @@ public class MetadataTableUtil {
}
public static void removeUnusedWALEntries(ServerContext context, KeyExtent
extent,
- final List<LogEntry> entries, ServiceLock zooLock) {
+ final Collection<LogEntry> entries, ServiceLock zooLock) {
TabletMutator tablet = context.getAmple().mutateTablet(extent);
entries.forEach(tablet::deleteWal);
tablet.putZooLock(context.getZooKeeperRoot(), zooLock);
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index 636156a624..8eeb7da704 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -98,7 +98,6 @@ import
org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
import org.apache.accumulo.core.tabletserver.thrift.TCompactionKind;
import org.apache.accumulo.core.tabletserver.thrift.TCompactionStats;
import org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob;
-import org.apache.accumulo.core.tabletserver.thrift.TTabletRefresh;
import org.apache.accumulo.core.tabletserver.thrift.TabletServerClientService;
import org.apache.accumulo.core.util.Retry;
import org.apache.accumulo.core.util.UtilWaitThread;
@@ -239,7 +238,7 @@ public class CompactionCoordinator implements
CompactionCoordinatorService.Iface
tablets.stream().forEach(tm -> tabletsMeta.put(tm.getExtent(), tm));
}
- var tserverRefreshes = new
HashMap<TabletMetadata.Location,List<TTabletRefresh>>();
+ var tserverRefreshes = new
HashMap<TabletMetadata.Location,List<TKeyExtent>>();
refreshEntries.forEach(refreshEntry -> {
var tm = tabletsMeta.get(refreshEntry.getExtent());
@@ -249,7 +248,7 @@ public class CompactionCoordinator implements
CompactionCoordinatorService.Iface
&&
tm.getLocation().getServerInstance().equals(refreshEntry.getTserver())) {
KeyExtent extent = tm.getExtent();
Collection<StoredTabletFile> scanfiles = tm.getScans();
- var ttr = TabletRefresher.createThriftRefresh(extent, scanfiles);
+ var ttr = extent.toThrift();
tserverRefreshes.computeIfAbsent(tm.getLocation(), k -> new
ArrayList<>()).add(ttr);
}
});
@@ -779,7 +778,7 @@ public class CompactionCoordinator implements
CompactionCoordinatorService.Iface
}
if (ecm.getKind() != CompactionKind.USER) {
- refreshTablet(tabletMeta, ecm.getJobFiles());
+ refreshTablet(tabletMeta);
}
// if a refresh entry was written, it can be removed after the tablet was
refreshed
@@ -811,11 +810,10 @@ public class CompactionCoordinator implements
CompactionCoordinatorService.Iface
}
}
- private void refreshTablet(TabletMetadata metadata,
Collection<StoredTabletFile> scanfiles) {
+ private void refreshTablet(TabletMetadata metadata) {
var location = metadata.getLocation();
if (location != null) {
KeyExtent extent = metadata.getExtent();
- TTabletRefresh tTabletRefresh =
TabletRefresher.createThriftRefresh(extent, scanfiles);
// there is a single tserver and single tablet, do not need a thread
pool. The direct executor
// will run everything in the current thread
@@ -823,7 +821,7 @@ public class CompactionCoordinator implements
CompactionCoordinatorService.Iface
try {
TabletRefresher.refreshTablets(executorService,
"compaction:" + metadata.getExtent().toString(), ctx,
tserverSet::getCurrentServers,
- Map.of(metadata.getLocation(), List.of(tTabletRefresh)));
+ Map.of(metadata.getLocation(), List.of(extent.toThrift())));
} finally {
executorService.shutdownNow();
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java
index 4ad5ccee1f..57d3616482 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java
@@ -25,7 +25,6 @@ import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -36,20 +35,16 @@ import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.function.Predicate;
import java.util.function.Supplier;
-import java.util.stream.Collectors;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.fate.FateTxId;
-import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
-import org.apache.accumulo.core.tabletserver.thrift.TTabletRefresh;
import org.apache.accumulo.core.tabletserver.thrift.TabletServerClientService;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.Retry;
@@ -74,8 +69,7 @@ public class TabletRefresher {
try (var tablets = context.getAmple().readTablets().forTable(tableId)
.overlapping(startRow, endRow).checkConsistency()
- .fetch(ColumnType.LOADED, ColumnType.LOCATION, ColumnType.PREV_ROW,
ColumnType.SCANS)
- .build()) {
+ .fetch(ColumnType.LOADED, ColumnType.LOCATION,
ColumnType.PREV_ROW).build()) {
// Find all tablets that need to refresh their metadata. There may be
some tablets that were
// hosted after the tablet files were updated, it just results in an
unneeded refresh
@@ -89,10 +83,8 @@ public class TabletRefresher {
// avoid reading all tablets into memory and instead process batches of
1000 tablets at a time
Iterators.partition(tabletIterator, 1000).forEachRemaining(batch -> {
- var refreshesNeeded = batch.stream()
- .collect(groupingBy(TabletMetadata::getLocation,
- mapping(tabletMetadata ->
createThriftRefresh(tabletMetadata.getExtent(),
- tabletMetadata.getScans()), toList())));
+ var refreshesNeeded =
batch.stream().collect(groupingBy(TabletMetadata::getLocation,
+ mapping(tabletMetadata -> tabletMetadata.getExtent().toThrift(),
toList())));
refreshTablets(threadPool, FateTxId.formatTid(fateTxid), context,
onlineTserversSupplier,
refreshesNeeded);
@@ -106,7 +98,7 @@ public class TabletRefresher {
public static void refreshTablets(ExecutorService threadPool, String logId,
ServerContext context,
Supplier<Set<TServerInstance>> onlineTserversSupplier,
- Map<TabletMetadata.Location,List<TTabletRefresh>> refreshesNeeded) {
+ Map<TabletMetadata.Location,List<TKeyExtent>> refreshesNeeded) {
// make a copy as it will be mutated in this method
refreshesNeeded = new HashMap<>(refreshesNeeded);
@@ -117,27 +109,25 @@ public class TabletRefresher {
while (!refreshesNeeded.isEmpty()) {
- Map<TabletMetadata.Location,Future<List<TTabletRefresh>>> futures = new
HashMap<>();
+ Map<TabletMetadata.Location,Future<List<TKeyExtent>>> futures = new
HashMap<>();
- for (Map.Entry<TabletMetadata.Location,List<TTabletRefresh>> entry :
refreshesNeeded
- .entrySet()) {
+ for (Map.Entry<TabletMetadata.Location,List<TKeyExtent>> entry :
refreshesNeeded.entrySet()) {
// Ask tablet server to reload the metadata for these tablets. The
tablet server returns
// the list of extents it was hosting but was unable to refresh (the
tablets could be in
// the process of loading). If it is not currently hosting the tablet
it treats that as
// refreshed and does not return anything for it.
- Future<List<TTabletRefresh>> future = threadPool
+ Future<List<TKeyExtent>> future = threadPool
.submit(() -> sendSyncRefreshRequest(context, logId,
entry.getKey(), entry.getValue()));
futures.put(entry.getKey(), future);
}
- for (Map.Entry<TabletMetadata.Location,Future<List<TTabletRefresh>>>
entry : futures
- .entrySet()) {
+ for (Map.Entry<TabletMetadata.Location,Future<List<TKeyExtent>>> entry :
futures.entrySet()) {
TabletMetadata.Location location = entry.getKey();
- Future<List<TTabletRefresh>> future = entry.getValue();
+ Future<List<TKeyExtent>> future = entry.getValue();
- List<TTabletRefresh> nonRefreshedExtents = null;
+ List<TKeyExtent> nonRefreshedExtents = null;
try {
nonRefreshedExtents = future.get();
} catch (InterruptedException | ExecutionException e) {
@@ -171,8 +161,8 @@ public class TabletRefresher {
}
}
- private static List<TTabletRefresh> sendSyncRefreshRequest(ServerContext
context, String logId,
- TabletMetadata.Location location, List<TTabletRefresh> refreshes) {
+ private static List<TKeyExtent> sendSyncRefreshRequest(ServerContext
context, String logId,
+ TabletMetadata.Location location, List<TKeyExtent> refreshes) {
TabletServerClientService.Client client = null;
try {
log.trace("{} sending refresh request to {} for {} extents", logId,
location,
@@ -186,14 +176,7 @@ public class TabletRefresher {
log.trace("{} refresh request to {} returned {} unrefreshed extents",
logId, location,
unrefreshed.size());
- if (unrefreshed.isEmpty()) {
- return List.of();
- }
-
- Map<TKeyExtent,TTabletRefresh> unrefreshedMap = new HashMap<>();
- refreshes.forEach(ttr -> unrefreshedMap.put(ttr.getExtent(), ttr));
- unrefreshedMap.keySet().retainAll(unrefreshed);
- return List.copyOf(unrefreshedMap.values());
+ return unrefreshed;
} catch (TException ex) {
log.debug("rpc failed server: " + location + ", " + logId + " " +
ex.getMessage(), ex);
@@ -206,10 +189,4 @@ public class TabletRefresher {
ThriftUtil.returnClient(client, context);
}
}
-
- public static TTabletRefresh createThriftRefresh(KeyExtent extent,
- Collection<StoredTabletFile> scanfiles) {
- return new TTabletRefresh(extent.toThrift(),
-
scanfiles.stream().map(StoredTabletFile::getMetaUpdateDelete).collect(Collectors.toList()));
- }
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
index 7d758abdd9..873934f90a 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
@@ -44,7 +44,6 @@ import org.apache.accumulo.server.problems.ProblemReports;
import
org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
import org.apache.accumulo.tserver.managermessage.TabletStatusMessage;
import org.apache.accumulo.tserver.tablet.Tablet;
-import org.apache.accumulo.tserver.tablet.TabletData;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -160,9 +159,8 @@ class AssignmentHandler implements Runnable {
TabletResourceManager trm =
server.resourceManager.createTabletResourceManager(extent,
server.getTableConfiguration(extent));
- TabletData data = new TabletData(tabletMetadata);
- tablet = new Tablet(server, extent, trm, data);
+ tablet = new Tablet(server, extent, trm, tabletMetadata);
// If a minor compaction starts after a tablet opens, this indicates a
log recovery
// occurred. This recovered data must be minor compacted.
// There are three reasons to wait for this minor compaction to finish
before placing the
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
index aacd1e61cd..ed18f0cd8a 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
@@ -75,7 +75,6 @@ import org.apache.accumulo.core.logging.TabletLogger;
import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.spi.cache.BlockCache;
@@ -89,7 +88,6 @@ import
org.apache.accumulo.core.tabletingest.thrift.TDurability;
import org.apache.accumulo.core.tabletingest.thrift.TabletIngestClientService;
import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
-import org.apache.accumulo.core.tabletserver.thrift.TTabletRefresh;
import org.apache.accumulo.core.tabletserver.thrift.TabletServerClientService;
import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
import org.apache.accumulo.core.trace.TraceUtil;
@@ -114,6 +112,7 @@ import org.apache.accumulo.tserver.session.UpdateSession;
import org.apache.accumulo.tserver.tablet.CommitSession;
import org.apache.accumulo.tserver.tablet.PreparedMutations;
import org.apache.accumulo.tserver.tablet.Tablet;
+import org.apache.accumulo.tserver.tablet.Tablet.RefreshPurpose;
import org.apache.accumulo.tserver.tablet.TabletClosedException;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.io.Text;
@@ -1144,7 +1143,7 @@ public class TabletClientHandler implements
TabletServerClientService.Iface,
@Override
public List<TKeyExtent> refreshTablets(TInfo tinfo, TCredentials credentials,
- List<TTabletRefresh> refreshes) throws TException {
+ List<TKeyExtent> refreshes) throws TException {
if (!security.canPerformSystemActions(credentials)) {
throw new AccumuloSecurityException(credentials.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED).asThriftException();
@@ -1154,10 +1153,10 @@ public class TabletClientHandler implements
TabletServerClientService.Iface,
// handle that more expensive case if needed.
var tabletsSnapshot = server.getOnlineTablets();
- Map<KeyExtent,TTabletRefresh> notFound = new HashMap<>();
+ Set<KeyExtent> notFound = new HashSet<>();
- for (var tTabletRefresh : refreshes) {
- var extent = KeyExtent.fromThrift(tTabletRefresh.getExtent());
+ for (var tkextent : refreshes) {
+ var extent = KeyExtent.fromThrift(tkextent);
var tablet = tabletsSnapshot.get(extent);
if (tablet != null) {
@@ -1166,10 +1165,9 @@ public class TabletClientHandler implements
TabletServerClientService.Iface,
// and multiple concurrent refresh request), so defer doing this until
after removing
// functionality from the tablet. No need to make the change now and
have to change it
// later.
- tablet.refresh(
-
tTabletRefresh.getScanEntries().stream().map(StoredTabletFile::new).collect(toList()));
+ tablet.refreshMetadata(RefreshPurpose.REFRESH_RPC);
} else {
- notFound.put(extent, tTabletRefresh);
+ notFound.add(extent);
}
}
@@ -1185,7 +1183,7 @@ public class TabletClientHandler implements
TabletServerClientService.Iface,
// Get the snapshot again, however this time nothing will be
changing while we iterate
// over the snapshot because all three locks are held.
tabletsSnapshot = server.getOnlineTablets();
- for (var extent : notFound.keySet()) {
+ for (var extent : notFound) {
// TODO investigate if its safe to ignore tablets in the
unopened set because they
// have not yet read any metadata
if (server.unopenedTablets.contains(extent)
@@ -1212,8 +1210,7 @@ public class TabletClientHandler implements
TabletServerClientService.Iface,
}
for (var tablet : foundTablets) {
-
tablet.refresh(notFound.get(tablet.getExtent()).getScanEntries().stream()
- .map(StoredTabletFile::new).collect(toList()));
+ tablet.refreshMetadata(RefreshPurpose.REFRESH_RPC);
}
if (log.isDebugEnabled()) {
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 51b6d4450e..0555aa72cf 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -19,10 +19,6 @@
package org.apache.accumulo.tserver;
import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
import static
org.apache.accumulo.core.util.threads.ThreadPools.watchCriticalFixedDelay;
import static
org.apache.accumulo.core.util.threads.ThreadPools.watchCriticalScheduledTask;
@@ -32,8 +28,6 @@ import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
-import java.time.Duration;
-import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -46,7 +40,6 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Optional;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -96,7 +89,6 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.Ample.TabletsMutator;
-import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
@@ -105,7 +97,6 @@ import
org.apache.accumulo.core.spi.ondemand.OnDemandTabletUnloader;
import
org.apache.accumulo.core.spi.ondemand.OnDemandTabletUnloader.UnloaderParams;
import org.apache.accumulo.core.tabletserver.UnloaderParamsImpl;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
-import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.ComparablePair;
import org.apache.accumulo.core.util.Halt;
import org.apache.accumulo.core.util.MapCounter;
@@ -151,7 +142,6 @@ import org.apache.accumulo.tserver.scan.ScanRunState;
import org.apache.accumulo.tserver.session.Session;
import org.apache.accumulo.tserver.session.SessionManager;
import org.apache.accumulo.tserver.tablet.CommitSession;
-import org.apache.accumulo.tserver.tablet.MetadataUpdateCount;
import org.apache.accumulo.tserver.tablet.Tablet;
import org.apache.commons.collections4.map.LRUMap;
import org.apache.hadoop.fs.Path;
@@ -166,7 +156,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.HostAndPort;
-import io.opentelemetry.api.trace.Span;
import io.opentelemetry.context.Scope;
public class TabletServer extends AbstractServer implements
TabletHostingServer {
@@ -686,54 +675,6 @@ public class TabletServer extends AbstractServer
implements TabletHostingServer
evaluateOnDemandTabletsForUnload();
});
- long tabletCheckFrequency =
aconf.getTimeInMillis(Property.TSERV_HEALTH_CHECK_FREQ);
- // Periodically check that metadata of tablets matches what is held in
memory
- watchCriticalFixedDelay(aconf, tabletCheckFrequency, () -> {
- final SortedMap<KeyExtent,Tablet> onlineTabletsSnapshot =
onlineTablets.snapshot();
-
- Map<KeyExtent,MetadataUpdateCount> updateCounts = new HashMap<>();
-
- // gather updateCounts for each tablet before reading tablet metadata
- onlineTabletsSnapshot.forEach((ke, tablet) -> {
- updateCounts.put(ke, tablet.getUpdateCount());
- });
-
- Instant start = Instant.now();
- Duration duration;
- Span mdScanSpan = TraceUtil.startSpan(this.getClass(), "metadataScan");
- try (Scope scope = mdScanSpan.makeCurrent()) {
- List<KeyExtent> missingTablets = new ArrayList<>();
- // gather metadata for all tablets readTablets()
- try (TabletsMetadata tabletsMetadata =
getContext().getAmple().readTablets()
- .forTablets(onlineTabletsSnapshot.keySet(),
Optional.of(missingTablets::add))
- .fetch(FILES, LOGS, ECOMP, PREV_ROW).build()) {
- duration = Duration.between(start, Instant.now());
- log.debug("Metadata scan took {}ms for {} tablets read.",
duration.toMillis(),
- onlineTabletsSnapshot.keySet().size());
-
- // for each tablet, compare its metadata to what is held in memory
- for (var tabletMetadata : tabletsMetadata) {
- KeyExtent extent = tabletMetadata.getExtent();
- Tablet tablet = onlineTabletsSnapshot.get(extent);
- MetadataUpdateCount counter = updateCounts.get(extent);
- tablet.compareTabletInfo(counter, tabletMetadata);
- }
-
- for (var extent : missingTablets) {
- Tablet tablet = onlineTabletsSnapshot.get(extent);
- if (!tablet.isClosed()) {
- log.error("Tablet {} is open but does not exist in metadata
table.", extent);
- }
- }
- }
- } catch (Exception e) {
- log.error("Unable to complete verification of tablet metadata", e);
- TraceUtil.setException(mdScanSpan, e, true);
- } finally {
- mdScanSpan.end();
- }
- });
-
HostAndPort managerHost;
while (!serverStopRequested) {
// send all of the pending messages
@@ -1002,7 +943,7 @@ public class TabletServer extends AbstractServer
implements TabletHostingServer
logger.minorCompactionStarted(tablet, lastUpdateSequence,
newDataFileLocation, durability);
}
- public void recover(VolumeManager fs, KeyExtent extent, List<LogEntry>
logEntries,
+ public void recover(VolumeManager fs, KeyExtent extent, Collection<LogEntry>
logEntries,
Set<String> tabletFiles, MutationReceiver mutationReceiver) throws
IOException {
List<Path> recoveryDirs = new ArrayList<>();
List<LogEntry> sorted = new ArrayList<>(logEntries);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
deleted file mode 100644
index f7f707202a..0000000000
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.tserver.tablet;
-
-import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.logging.TabletLogger;
-import org.apache.accumulo.core.metadata.ReferencedTabletFile;
-import org.apache.accumulo.core.metadata.StoredTabletFile;
-import org.apache.accumulo.core.metadata.schema.DataFileValue;
-import org.apache.accumulo.core.util.MapCounter;
-import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.util.MetadataTableUtil;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-class DatafileManager {
- private final Logger log = LoggerFactory.getLogger(DatafileManager.class);
- // access to datafilesizes needs to be synchronized: see
CompactionRunner#getNumFiles
- private final Map<StoredTabletFile,DataFileValue> datafileSizes =
- Collections.synchronizedMap(new TreeMap<>());
- private final Tablet tablet;
-
- // This must be incremented before and after datafileSizes and metadata
table updates. These
- // counts allow detection of overlapping operations w/o placing a lock
around metadata table
- // updates and datafileSizes updates. There is a periodic metadata
consistency check that runs in
- // the tablet server against all tablets. This check compares what a tablet
object has in memory
- // to what is in the metadata table to ensure they are in agreement. Inorder
to avoid false
- // positives, when this consistency check runs its needs to know if it
overlaps in time with any
- // metadata updates made by the tablet. The consistency check uses these
counts to know that.
- private final AtomicReference<MetadataUpdateCount> metadataUpdateCount;
-
- DatafileManager(Tablet tablet, SortedMap<StoredTabletFile,DataFileValue>
datafileSizes) {
- this.datafileSizes.putAll(datafileSizes);
- this.tablet = tablet;
- this.metadataUpdateCount =
- new AtomicReference<>(new MetadataUpdateCount(tablet.getExtent(), 0L,
0L));
- }
-
- private final Set<StoredTabletFile> filesToDeleteAfterScan = new HashSet<>();
- private final Map<Long,Set<StoredTabletFile>> scanFileReservations = new
HashMap<>();
- private final MapCounter<StoredTabletFile> fileScanReferenceCounts = new
MapCounter<>();
- private long nextScanReservationId = 0;
-
- static void rename(VolumeManager fs, Path src, Path dst) throws IOException {
- if (!fs.rename(src, dst)) {
- throw new IOException("Rename " + src + " to " + dst + " returned false
");
- }
- }
-
- Pair<Long,Map<StoredTabletFile,DataFileValue>> reserveFilesForScan() {
- synchronized (tablet) {
-
- Set<StoredTabletFile> absFilePaths = new
HashSet<>(datafileSizes.keySet());
-
- long rid = nextScanReservationId++;
-
- scanFileReservations.put(rid, absFilePaths);
-
- Map<StoredTabletFile,DataFileValue> ret = new HashMap<>();
-
- for (StoredTabletFile path : absFilePaths) {
- fileScanReferenceCounts.increment(path, 1);
- ret.put(path, datafileSizes.get(path));
- }
-
- return new Pair<>(rid, ret);
- }
- }
-
- void returnFilesForScan(Long reservationId) {
-
- final Set<StoredTabletFile> filesToDelete = new HashSet<>();
-
- synchronized (tablet) {
- Set<StoredTabletFile> absFilePaths =
scanFileReservations.remove(reservationId);
-
- if (absFilePaths == null) {
- throw new IllegalArgumentException("Unknown scan reservation id " +
reservationId);
- }
-
- boolean notify = false;
- for (StoredTabletFile path : absFilePaths) {
- long refCount = fileScanReferenceCounts.decrement(path, 1);
- if (refCount == 0) {
- if (filesToDeleteAfterScan.remove(path)) {
- filesToDelete.add(path);
- }
- notify = true;
- } else if (refCount < 0) {
- throw new IllegalStateException("Scan ref count for " + path + " is
" + refCount);
- }
- }
-
- if (notify) {
- tablet.notifyAll();
- }
- }
-
- if (!filesToDelete.isEmpty()) {
- log.debug("Removing scan refs from metadata {} {}", tablet.getExtent(),
filesToDelete);
- MetadataTableUtil.removeScanFiles(tablet.getExtent(), filesToDelete,
tablet.getContext(),
- tablet.getTabletServer().getLock());
- }
- }
-
- void removeFilesAfterScan(Set<StoredTabletFile> scanFiles) {
- if (scanFiles.isEmpty()) {
- return;
- }
-
- Set<StoredTabletFile> filesToDelete = new HashSet<>();
-
- synchronized (tablet) {
- for (StoredTabletFile path : scanFiles) {
- if (fileScanReferenceCounts.get(path) == 0) {
- filesToDelete.add(path);
- } else {
- filesToDeleteAfterScan.add(path);
- }
- }
- }
-
- if (!filesToDelete.isEmpty()) {
- log.debug("Removing scan refs from metadata {} {}", tablet.getExtent(),
filesToDelete);
- MetadataTableUtil.removeScanFiles(tablet.getExtent(), filesToDelete,
tablet.getContext(),
- tablet.getTabletServer().getLock());
- }
- }
-
- /**
- * Returns Optional of the new file created. It is possible that the file
was just flushed with no
- * entries so was not inserted into the metadata. In this case empty is
returned. If the file was
- * stored in the metadata table, then StoredTableFile will be returned.
- */
- Optional<StoredTabletFile> bringMinorCompactionOnline(ReferencedTabletFile
tmpDatafile,
- ReferencedTabletFile newDatafile, DataFileValue dfv, CommitSession
commitSession,
- long flushId) {
- Optional<StoredTabletFile> newFile;
- // rename before putting in metadata table, so files in metadata table
should
- // always exist
- boolean attemptedRename = false;
- VolumeManager vm =
tablet.getTabletServer().getContext().getVolumeManager();
- do {
- try {
- if (dfv.getNumEntries() == 0) {
- log.debug("No data entries so delete temporary file {}",
tmpDatafile);
- vm.deleteRecursively(tmpDatafile.getPath());
- } else {
- if (!attemptedRename && vm.exists(newDatafile.getPath())) {
- log.warn("Target data file already exist {}", newDatafile);
- throw new RuntimeException("File unexpectedly exists " +
newDatafile.getPath());
- }
- // the following checks for spurious rename failures that succeeded
but gave an IoE
- if (attemptedRename && vm.exists(newDatafile.getPath())
- && !vm.exists(tmpDatafile.getPath())) {
- // seems like previous rename succeeded, so break
- break;
- }
- attemptedRename = true;
- rename(vm, tmpDatafile.getPath(), newDatafile.getPath());
- }
- break;
- } catch (IOException ioe) {
- log.warn("Tablet " + tablet.getExtent() + " failed to rename " +
newDatafile
- + " after MinC, will retry in 60 secs...", ioe);
- sleepUninterruptibly(1, TimeUnit.MINUTES);
- }
- } while (true);
-
- long t1, t2;
-
- // increment start count before metadata update AND updating in memory map
of files
- metadataUpdateCount.updateAndGet(MetadataUpdateCount::incrementStart);
- // do not place any code here between above stmt and try{}finally
- try {
- Set<String> unusedWalLogs = tablet.beginClearingUnusedLogs();
- try {
- // the order of writing to metadata and walog is important in the face
of machine/process
- // failures need to write to metadata before writing to walog, when
things are done in the
- // reverse order data could be lost... the minor compaction start even
should be written
- // before the following metadata write is made
- newFile =
tablet.updateTabletDataFile(commitSession.getMaxCommittedTime(), newDatafile,
dfv,
- unusedWalLogs, flushId);
- } finally {
- tablet.finishClearingUnusedLogs();
- }
-
- do {
- try {
- // the purpose of making this update use the new commit session,
instead of the old one
- // passed in, is because the new one will reference the logs used by
current memory...
-
- tablet.getTabletServer().minorCompactionFinished(
- tablet.getTabletMemory().getCommitSession(),
commitSession.getWALogSeq() + 2);
- break;
- } catch (IOException e) {
- log.error("Failed to write to write-ahead log " + e.getMessage() + "
will retry", e);
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- }
- } while (true);
-
- synchronized (tablet) {
- t1 = System.currentTimeMillis();
-
- if (newFile.isPresent()) {
- StoredTabletFile newFileStored = newFile.orElseThrow();
- if (datafileSizes.containsKey(newFileStored)) {
- log.error("Adding file that is already in set {}", newFileStored);
- }
- datafileSizes.put(newFileStored, dfv);
- }
-
- tablet.flushComplete(flushId);
-
- t2 = System.currentTimeMillis();
- }
- } finally {
- // increment finish count after metadata update AND updating in memory
map of files
- metadataUpdateCount.updateAndGet(MetadataUpdateCount::incrementFinish);
- }
-
- TabletLogger.flushed(tablet.getExtent(), newFile);
-
- if (log.isTraceEnabled()) {
- log.trace(String.format("MinC finish lock %.2f secs %s", (t2 - t1) /
1000.0,
- tablet.getExtent().toString()));
- }
- long splitSize =
tablet.getTableConfiguration().getAsBytes(Property.TABLE_SPLIT_THRESHOLD);
- if (dfv.getSize() > splitSize) {
- log.debug(String.format("Minor Compaction wrote out file larger than
split threshold."
- + " split threshold = %,d file size = %,d", splitSize,
dfv.getSize()));
- }
-
- return newFile;
- }
-
- public SortedMap<StoredTabletFile,DataFileValue> getDatafileSizes() {
- synchronized (tablet) {
- TreeMap<StoredTabletFile,DataFileValue> copy = new
TreeMap<>(datafileSizes);
- return Collections.unmodifiableSortedMap(copy);
- }
- }
-
- public MetadataUpdateCount getUpdateCount() {
- return metadataUpdateCount.get();
- }
-
- // ELASTICITY_TODO remove this method
- public void setFilesHack(Map<StoredTabletFile,DataFileValue> files) {
- synchronized (tablet) {
- datafileSizes.clear();
- datafileSizes.putAll(files);
- }
- }
-}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MetadataUpdateCount.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MetadataUpdateCount.java
deleted file mode 100644
index 42738a193e..0000000000
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MetadataUpdateCount.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.tserver.tablet;
-
-import java.util.Objects;
-
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-
-/**
- * The tablet server does periodic consistency checks to see if what is in the
metadata table agrees
- * with what each tablet has in memory. When doing these checks its very
important to know if the
- * tablet severs read from the metadata table overlaps in time with any tablet
metadata table
- * updates. These counts allow that to be known. For example if these counts
are acquired twice for
- * a tablet and are the same both times it means that no metadata table
updates occurred between the
- * two acquisition times.
- */
-public class MetadataUpdateCount {
- private final KeyExtent extent;
- private final long startedCount;
- private final long finishedCount;
-
- MetadataUpdateCount(KeyExtent extent, long startedCount, long finishedCount)
{
- this.extent = Objects.requireNonNull(extent);
- this.startedCount = startedCount;
- this.finishedCount = finishedCount;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- MetadataUpdateCount that = (MetadataUpdateCount) o;
- return startedCount == that.startedCount && finishedCount ==
that.finishedCount;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(startedCount, finishedCount);
- }
-
- public KeyExtent getExtent() {
- return extent;
- }
-
- /**
- * @return true if the counters were acquired while a metadata table update
was being made
- */
- public boolean overlapsUpdate() {
- return startedCount != finishedCount;
- }
-
- public MetadataUpdateCount incrementStart() {
- return new MetadataUpdateCount(extent, startedCount + 1, finishedCount);
- }
-
- public MetadataUpdateCount incrementFinish() {
- return new MetadataUpdateCount(extent, startedCount, finishedCount + 1);
- }
-
- @Override
- public String toString() {
- return "[startedCount:" + startedCount + ",finishedCount:" + finishedCount
+ "]";
- }
-}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
new file mode 100644
index 0000000000..4513dca855
--- /dev/null
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.tserver.tablet;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class ScanfileManager {
+ private final Logger log = LoggerFactory.getLogger(ScanfileManager.class);
+ private final Tablet tablet;
+
+ ScanfileManager(Tablet tablet) {
+ this.tablet = tablet;
+ }
+
+ private final Set<StoredTabletFile> filesToDeleteAfterScan = new HashSet<>();
+ private final Map<Long,Set<StoredTabletFile>> scanFileReservations = new
HashMap<>();
+ private final MapCounter<StoredTabletFile> fileScanReferenceCounts = new
MapCounter<>();
+ private long nextScanReservationId = 0;
+
+ static void rename(VolumeManager fs, Path src, Path dst) throws IOException {
+ if (!fs.rename(src, dst)) {
+ throw new IOException("Rename " + src + " to " + dst + " returned false
");
+ }
+ }
+
+ Pair<Long,Map<StoredTabletFile,DataFileValue>> reserveFilesForScan() {
+ synchronized (tablet) {
+
+ var tabletsFiles = tablet.getDatafiles();
+ Set<StoredTabletFile> absFilePaths = new
HashSet<>(tabletsFiles.keySet());
+
+ long rid = nextScanReservationId++;
+
+ scanFileReservations.put(rid, absFilePaths);
+
+ Map<StoredTabletFile,DataFileValue> ret = new HashMap<>();
+
+ for (StoredTabletFile path : absFilePaths) {
+ fileScanReferenceCounts.increment(path, 1);
+ ret.put(path, tabletsFiles.get(path));
+ }
+
+ return new Pair<>(rid, ret);
+ }
+ }
+
+ void returnFilesForScan(Long reservationId) {
+
+ final Set<StoredTabletFile> filesToDelete = new HashSet<>();
+
+ synchronized (tablet) {
+ Set<StoredTabletFile> absFilePaths =
scanFileReservations.remove(reservationId);
+
+ if (absFilePaths == null) {
+ throw new IllegalArgumentException("Unknown scan reservation id " +
reservationId);
+ }
+
+ boolean notify = false;
+ for (StoredTabletFile path : absFilePaths) {
+ long refCount = fileScanReferenceCounts.decrement(path, 1);
+ if (refCount == 0) {
+ if (filesToDeleteAfterScan.remove(path)) {
+ filesToDelete.add(path);
+ }
+ notify = true;
+ } else if (refCount < 0) {
+ throw new IllegalStateException("Scan ref count for " + path + " is
" + refCount);
+ }
+ }
+
+ if (notify) {
+ tablet.notifyAll();
+ }
+ }
+
+ if (!filesToDelete.isEmpty()) {
+ log.debug("Removing scan refs from metadata {} {}", tablet.getExtent(),
filesToDelete);
+ // ELASTICTIY_TODO use conditional mutation
+ MetadataTableUtil.removeScanFiles(tablet.getExtent(), filesToDelete,
tablet.getContext(),
+ tablet.getTabletServer().getLock());
+ }
+ }
+
+ void removeFilesAfterScan(Collection<StoredTabletFile> scanFiles) {
+ if (scanFiles.isEmpty()) {
+ return;
+ }
+
+ Set<StoredTabletFile> filesToDelete = new HashSet<>();
+
+ synchronized (tablet) {
+ for (StoredTabletFile path : scanFiles) {
+ if (fileScanReferenceCounts.get(path) == 0) {
+ filesToDelete.add(path);
+ } else {
+ filesToDeleteAfterScan.add(path);
+ }
+ }
+ }
+
+ if (!filesToDelete.isEmpty()) {
+ // ELASTICTIY_TODO use conditional mutation
+ log.debug("Removing scan refs from metadata {} {}", tablet.getExtent(),
filesToDelete);
+ MetadataTableUtil.removeScanFiles(tablet.getExtent(), filesToDelete,
tablet.getContext(),
+ tablet.getTabletServer().getLock());
+ }
+ }
+}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SnapshotTablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SnapshotTablet.java
index 57c7036902..967b6f137c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SnapshotTablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SnapshotTablet.java
@@ -67,7 +67,7 @@ public class SnapshotTablet extends TabletBase {
}
@Override
- public SortedMap<StoredTabletFile,DataFileValue> getDatafiles() {
+ public Map<StoredTabletFile,DataFileValue> getDatafiles() {
return files;
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index b7e959591f..7f776f7d09 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -33,11 +33,11 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.SortedMap;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -75,8 +75,7 @@ import
org.apache.accumulo.core.tabletserver.thrift.TabletStats;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.server.compaction.CompactionStats;
-import org.apache.accumulo.server.fs.VolumeUtil;
-import org.apache.accumulo.server.fs.VolumeUtil.TabletFiles;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.problems.ProblemReport;
import org.apache.accumulo.server.problems.ProblemReports;
import org.apache.accumulo.server.problems.ProblemType;
@@ -118,20 +117,18 @@ public class Tablet extends TabletBase {
private final TabletServer tabletServer;
private final TabletResourceManager tabletResources;
- private final DatafileManager datafileManager;
- private final String dirName;
-
+ private final ScanfileManager scanfileManager;
private final TabletMemory tabletMemory;
private final TabletTime tabletTime;
- private final Object timeLock = new Object();
- private long persistedTime;
private Location lastLocation = null;
private final Set<Path> checkedTabletDirs = new ConcurrentSkipListSet<>();
private final AtomicLong dataSourceDeletions = new AtomicLong(0);
+ private volatile TabletMetadata latestMetadata;
+
@Override
public long getDataSourceDeletions() {
return dataSourceDeletions.get();
@@ -145,8 +142,6 @@ public class Tablet extends TabletBase {
private boolean updatingFlushID = false;
- private final AtomicLong lastFlushID = new AtomicLong(-1);
-
enum CompactionState {
WAITING_TO_START, IN_PROGRESS
}
@@ -183,9 +178,6 @@ public class Tablet extends TabletBase {
private final int logId;
- // TODO: User can change this, how does it get updated?
- private final TabletHostingGoal goal;
-
public int getLogId() {
return logId;
}
@@ -198,8 +190,8 @@ public class Tablet extends TabletBase {
}
ReferencedTabletFile getNextDataFilename(FilePrefix prefix) throws
IOException {
- return TabletNameGenerator.getNextDataFilename(prefix, context, extent,
dirName,
- dir -> checkTabletDir(new Path(dir)));
+ return TabletNameGenerator.getNextDataFilename(prefix, context, extent,
+ getMetadata().getDirName(), dir -> checkTabletDir(new Path(dir)));
}
private void checkTabletDir(Path path) {
@@ -225,35 +217,27 @@ public class Tablet extends TabletBase {
}
public Tablet(final TabletServer tabletServer, final KeyExtent extent,
- final TabletResourceManager trm, TabletData data)
+ final TabletResourceManager trm, TabletMetadata metadata)
throws IOException, IllegalArgumentException {
super(tabletServer, extent);
this.tabletServer = tabletServer;
this.tabletResources = trm;
- this.lastLocation = data.getLastLocation();
- this.lastFlushID.set(data.getFlushID());
- this.tabletTime = TabletTime.getInstance(data.getTime());
- this.persistedTime = tabletTime.getTime();
- this.logId = tabletServer.createLogId();
- this.goal = data.getHostingGoal();
+ this.latestMetadata = metadata;
- // translate any volume changes
- TabletFiles tabletPaths =
- new TabletFiles(data.getDirectoryName(), data.getLogEntries(),
data.getDataFiles());
- tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer.getContext(),
tabletServer.getLock(),
- extent, tabletPaths);
+ // TODO look into this.. also last could be null
+ this.lastLocation = metadata.getLast();
- this.dirName = data.getDirectoryName();
-
- final List<LogEntry> logEntries = tabletPaths.logEntries;
- final SortedMap<StoredTabletFile,DataFileValue> datafiles =
tabletPaths.datafiles;
+ this.tabletTime = TabletTime.getInstance(metadata.getTime());
+ this.logId = tabletServer.createLogId();
constraintChecker = tableConfiguration.newDeriver(ConstraintChecker::new);
tabletMemory = new TabletMemory(this);
+ var logEntries = new ArrayList<>(metadata.getLogs());
+
// don't bother examining WALs for recovery if Table is being deleted
if (!logEntries.isEmpty() && !isBeingDeleted()) {
TabletLogger.recovering(extent, logEntries);
@@ -263,7 +247,7 @@ public class Tablet extends TabletBase {
final CommitSession commitSession = getTabletMemory().getCommitSession();
try {
Set<String> absPaths = new HashSet<>();
- for (StoredTabletFile ref : datafiles.keySet()) {
+ for (StoredTabletFile ref : metadata.getFiles()) {
absPaths.add(ref.getNormalizedPathStr());
}
@@ -288,8 +272,11 @@ public class Tablet extends TabletBase {
if (entriesUsedOnTablet.get() == 0) {
log.debug("No replayed mutations applied, removing unused entries
for {}", extent);
+ // ELASTICITY_TODO use conditional mutation for update
MetadataTableUtil.removeUnusedWALEntries(getTabletServer().getContext(), extent,
logEntries, tabletServer.getLock());
+ // intentionally not rereading metadata here because walogs are only
used in the
+ // constructor
logEntries.clear();
}
@@ -316,11 +303,15 @@ public class Tablet extends TabletBase {
// do this last after tablet is completely setup because it
// could cause major compaction to start
- datafileManager = new DatafileManager(this, datafiles);
+ scanfileManager = new ScanfileManager(this);
computeNumEntries();
- getDatafileManager().removeFilesAfterScan(data.getScanFiles());
+ getScanfileManager().removeFilesAfterScan(metadata.getScans());
+ }
+
+ public TabletMetadata getMetadata() {
+ return latestMetadata;
}
public void checkConditions(ConditionChecker checker, Authorizations
authorizations,
@@ -375,7 +366,7 @@ public class Tablet extends TabletBase {
Span span2 = TraceUtil.startSpan(this.getClass(),
"minorCompact::bringOnline");
try (Scope scope = span2.makeCurrent()) {
- getDatafileManager().bringMinorCompactionOnline(tmpDatafile,
newDatafile,
+ bringMinorCompactionOnline(tmpDatafile, newDatafile,
new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()),
commitSession,
flushId);
} catch (Exception e) {
@@ -433,7 +424,7 @@ public class Tablet extends TabletBase {
return;
}
- if (lastFlushID.get() >= tableFlushID) {
+ if (getMetadata().getFlushId().orElse(-1) >= tableFlushID) {
return;
}
@@ -443,7 +434,6 @@ public class Tablet extends TabletBase {
}
if (getTabletMemory().getMemTable().getNumEntries() == 0) {
- lastFlushID.set(tableFlushID);
updatingFlushID = true;
updateMetadata = true;
} else {
@@ -452,10 +442,21 @@ public class Tablet extends TabletBase {
}
if (updateMetadata) {
- // if multiple threads were allowed to update this outside of a sync
block, then it would be
- // a race condition
- MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, context,
- getTabletServer().getLock());
+ refreshLock.lock();
+ try {
+ // if multiple threads were allowed to update this outside of a sync
block, then it would
+ // be
+ // a race condition
+ // ELASTICITY_TODO use conditional mutations
+ MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, context,
+ getTabletServer().getLock());
+ // It is important the the refresh lock is held for the update above
and the refresh below
+ // to avoid race conditions.
+ refreshMetadata(RefreshPurpose.FLUSH_ID_UPDATE);
+ } finally {
+ refreshLock.unlock();
+ }
+
} else if (initiateMinor) {
initiateMinorCompaction(tableFlushID, MinorCompactionReason.USER);
}
@@ -901,21 +902,6 @@ public class Tablet extends TabletBase {
log.error(msg);
throw new RuntimeException(msg);
}
-
- tabletMeta.getFlushId().ifPresent(flushId -> {
- if (flushId != lastFlushID.get()) {
- String msg = "Closed tablet " + extent + " lastFlushID is
inconsistent with metadata : "
- + flushId + " != " + lastFlushID;
- log.error(msg);
- throw new RuntimeException(msg);
- }
- });
-
- if
(!tabletMeta.getFilesMap().equals(getDatafileManager().getDatafileSizes())) {
- String msg = "Data files in " + extent + " differ from in-memory data "
- + tabletMeta.getFilesMap() + " " +
getDatafileManager().getDatafileSizes();
- log.error(msg);
- }
} catch (Exception e) {
String msg = "Failed to do close consistency check for tablet " + extent;
log.error(msg, e);
@@ -931,72 +917,8 @@ public class Tablet extends TabletBase {
}
}
- private boolean loggedErrorForTabletComparison = false;
-
- /**
- * Checks that tablet metadata from the metadata table matches what this
tablet has in memory. The
- * caller of this method must acquire the updateCounter parameter before
acquiring the
- * tabletMetadata.
- *
- * @param updateCounter used to check for conucurrent updates in which case
this check is a no-op.
- * See {@link #getUpdateCount()}
- * @param tabletMetadata the metadata for this tablet that was acquired from
the metadata table.
- */
- public synchronized void compareTabletInfo(MetadataUpdateCount updateCounter,
- TabletMetadata tabletMetadata) {
-
- // verify the given counter is for this tablet, if this check fail it
indicates a bug in the
- // calling code
- Preconditions.checkArgument(updateCounter.getExtent().equals(getExtent()),
- "Counter had unexpected extent %s != %s", updateCounter.getExtent(),
getExtent());
-
- // verify the given tablet metadata is for this tablet, if this check fail
it indicates a bug in
- // the calling code
- Preconditions.checkArgument(tabletMetadata.getExtent().equals(getExtent()),
- "Tablet metadata had unexpected extent %s != %s",
tabletMetadata.getExtent(), getExtent());
-
- // All of the log messages in this method have the AMCC acronym which
means Accumulo Metadata
- // Consistency Check. AMCC was added to the log messages to make
grep/search for all log
- // message from this method easy to find.
-
- if (isClosed() || isClosing()) {
- log.trace("AMCC Tablet {} was closed, so skipping check",
tabletMetadata.getExtent());
- return;
- }
-
- var dataFileSizes = getDatafileManager().getDatafileSizes();
-
- if (!tabletMetadata.getFilesMap().equals(dataFileSizes)) {
- // The counters are modified outside of locks before and after tablet
metadata operations and
- // data file updates so, it's very important to acquire the 2nd counts
after doing the
- // equality check above. If the counts are the same (as the ones
acquired before reading
- // metadata table) after the equality check above then we know the
tablet did not do any
- // metadata updates while we were reading metadata and then comparing.
- var latestCount = this.getUpdateCount();
- if (updateCounter.overlapsUpdate() ||
!updateCounter.equals(latestCount)) {
- log.trace(
- "AMCC Tablet {} may have been updating its metadata while it was
being read for "
- + "check, so skipping check {} {}",
- tabletMetadata.getExtent(), updateCounter, latestCount);
- } else {
- log.error("Data files in {} differ from in-memory data {} {} {} {}",
extent,
- tabletMetadata.getFilesMap(), dataFileSizes, updateCounter,
latestCount);
- loggedErrorForTabletComparison = true;
- }
- } else {
- if (loggedErrorForTabletComparison) {
- log.info("AMCC Tablet {} files in memory are now same as in metadata
table {}",
- tabletMetadata.getExtent(), updateCounter);
- loggedErrorForTabletComparison = false;
- } else {
- log.trace("AMCC Tablet {} files in memory are same as in metadata
table {}",
- tabletMetadata.getExtent(), updateCounter);
- }
- }
- }
-
synchronized void computeNumEntries() {
- Collection<DataFileValue> vals =
getDatafileManager().getDatafileSizes().values();
+ Collection<DataFileValue> vals = getDatafiles().values();
long numEntries = 0;
@@ -1048,8 +970,8 @@ public class Tablet extends TabletBase {
}
@Override
- public SortedMap<StoredTabletFile,DataFileValue> getDatafiles() {
- return getDatafileManager().getDatafileSizes();
+ public Map<StoredTabletFile,DataFileValue> getDatafiles() {
+ return getMetadata().getFilesMap();
}
@Override
@@ -1325,17 +1247,21 @@ public class Tablet extends TabletBase {
public Optional<StoredTabletFile> updateTabletDataFile(long maxCommittedTime,
ReferencedTabletFile newDatafile, DataFileValue dfv, Set<String>
unusedWalLogs,
long flushId) {
- synchronized (timeLock) {
- if (maxCommittedTime > persistedTime) {
- persistedTime = maxCommittedTime;
- }
- return
ManagerMetadataUtil.updateTabletDataFile(getTabletServer().getContext(), extent,
- newDatafile, dfv, tabletTime.getMetadataTime(persistedTime),
- tabletServer.getTabletSession(), tabletServer.getLock(),
unusedWalLogs, lastLocation,
- flushId);
- }
+ // expect time to only move forward from what was recently seen in
metadata table
+ Preconditions.checkArgument(maxCommittedTime >=
getMetadata().getTime().getTime());
+
+ // ELASTICITY_TODO use conditional mutation, can check time and location
+
+ // ELASTICITY_TODO minor compaction will need something like the bulk
import loaded column
+ // to avoid : partial write, compact of file in partial write, and then
another write of the
+ // file
+ // leading to the file being added twice.
+ return
ManagerMetadataUtil.updateTabletDataFile(getTabletServer().getContext(), extent,
+ newDatafile, dfv, tabletTime.getMetadataTime(maxCommittedTime),
+ tabletServer.getTabletSession(), tabletServer.getLock(),
unusedWalLogs, lastLocation,
+ flushId);
}
@Override
@@ -1348,22 +1274,18 @@ public class Tablet extends TabletBase {
return getTabletServer().getScanMetrics();
}
- DatafileManager getDatafileManager() {
- return datafileManager;
+ ScanfileManager getScanfileManager() {
+ return scanfileManager;
}
@Override
public Pair<Long,Map<StoredTabletFile,DataFileValue>> reserveFilesForScan() {
- return getDatafileManager().reserveFilesForScan();
+ return getScanfileManager().reserveFilesForScan();
}
@Override
public void returnFilesForScan(long scanId) {
- getDatafileManager().returnFilesForScan(scanId);
- }
-
- public MetadataUpdateCount getUpdateCount() {
- return getDatafileManager().getUpdateCount();
+ getScanfileManager().returnFilesForScan(scanId);
}
TabletMemory getTabletMemory() {
@@ -1380,14 +1302,6 @@ public class Tablet extends TabletBase {
getTabletMemory().returnIterators(iters);
}
- public void flushComplete(long flushId) {
- lastLocation = null;
- dataSourceDeletions.incrementAndGet();
- tabletMemory.finishedMinC();
- lastFlushID.set(flushId);
- computeNumEntries();
- }
-
public void minorCompactionWaitingToStart() {
minorCompactionState = CompactionState.WAITING_TO_START;
}
@@ -1405,48 +1319,143 @@ public class Tablet extends TabletBase {
}
public boolean isOnDemand() {
- return goal == TabletHostingGoal.ONDEMAND;
+ // TODO a change in the hosting goal could refresh online tablets
+ return getMetadata().getHostingGoal() == TabletHostingGoal.ONDEMAND;
}
- public void refresh(List<StoredTabletFile> scanEntries) {
- if (isClosing() || isClosed()) {
- // TODO this is just a best effort could close after this check, its a
race condition.
- // Intentionally not being handled ATM.
- return;
+ // The purpose of this lock is to prevent race conditions between concurrent
refresh RPC calls and
+ // between minor compactions and refresh calls.
+ private final Lock refreshLock = new ReentrantLock();
+
+ void bringMinorCompactionOnline(ReferencedTabletFile tmpDatafile,
+ ReferencedTabletFile newDatafile, DataFileValue dfv, CommitSession
commitSession,
+ long flushId) {
+ Optional<StoredTabletFile> newFile;
+ // rename before putting in metadata table, so files in metadata table
should
+ // always exist
+ boolean attemptedRename = false;
+ VolumeManager vm = getTabletServer().getContext().getVolumeManager();
+ do {
+ try {
+ if (dfv.getNumEntries() == 0) {
+ log.debug("No data entries so delete temporary file {}",
tmpDatafile);
+ vm.deleteRecursively(tmpDatafile.getPath());
+ } else {
+ if (!attemptedRename && vm.exists(newDatafile.getPath())) {
+ log.warn("Target data file already exist {}", newDatafile);
+ throw new RuntimeException("File unexpectedly exists " +
newDatafile.getPath());
+ }
+ // the following checks for spurious rename failures that succeeded
but gave an IoE
+ if (attemptedRename && vm.exists(newDatafile.getPath())
+ && !vm.exists(tmpDatafile.getPath())) {
+ // seems like previous rename succeeded, so break
+ break;
+ }
+ attemptedRename = true;
+ ScanfileManager.rename(vm, tmpDatafile.getPath(),
newDatafile.getPath());
+ }
+ break;
+ } catch (IOException ioe) {
+ log.warn("Tablet " + getExtent() + " failed to rename " + newDatafile
+ + " after MinC, will retry in 60 secs...", ioe);
+ sleepUninterruptibly(1, TimeUnit.MINUTES);
+ }
+ } while (true);
+
+ // The refresh lock must be held for the metadata write that adds the new
file to the tablet.
+ // This prevents a concurrent refresh operation from pulling in the new
tablet file before the
+ // in memory map reference related to the file is deactivated. Scans
should use one of the in
+ // memory map or the new file, never both.
+ refreshLock.lock();
+ try {
+ Set<String> unusedWalLogs = beginClearingUnusedLogs();
+ try {
+ // the order of writing to metadata and walog is important in the face
of machine/process
+ // failures need to write to metadata before writing to walog, when
things are done in the
+ // reverse order data could be lost... the minor compaction start
event should be written
+ // before the following metadata write is made
+
+ newFile = updateTabletDataFile(commitSession.getMaxCommittedTime(),
newDatafile, dfv,
+ unusedWalLogs, flushId);
+ } finally {
+ finishClearingUnusedLogs();
+ }
+
+ // Without the refresh lock, if a refresh happened here it could make
the new file written to
+ // the metadata table above available for scans while the in memory map
from which the file
+ // was produced is still available for scans
+
+ do {
+ try {
+ // the purpose of making this update use the new commit session,
instead of the old one
+ // passed in, is because the new one will reference the logs used by
current memory...
+
getTabletServer().minorCompactionFinished(getTabletMemory().getCommitSession(),
+ commitSession.getWALogSeq() + 2);
+ break;
+ } catch (IOException e) {
+ log.error("Failed to write to write-ahead log " + e.getMessage() + "
will retry", e);
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ }
+ } while (true);
+
+ refreshMetadata(RefreshPurpose.MINC_COMPLETION);
+ } finally {
+ refreshLock.unlock();
}
+ TabletLogger.flushed(getExtent(), newFile);
+
+ long splitSize =
getTableConfiguration().getAsBytes(Property.TABLE_SPLIT_THRESHOLD);
+ if (dfv.getSize() > splitSize) {
+ log.debug(String.format("Minor Compaction wrote out file larger than
split threshold."
+ + " split threshold = %,d file size = %,d", splitSize,
dfv.getSize()));
+ }
+ }
- // ELASTICITY_TODO instead of reading the tablet metadata in this method,
could just invalidate
- // it forcing next scan to read it.
+ public enum RefreshPurpose {
+ MINC_COMPLETION, REFRESH_RPC, FLUSH_ID_UPDATE
+ }
- // ELASTICITY_TODO this entire method is a hack at the moment with race
conditions. Want to
- // move towards the tablet just using a cached TabletMetadata object and
have a central orderly
- // thread safe way to update it within the tablet in response to external
refresh request and
- // internal events like minor compactions. Would probably be easiest to
implement this after
- // removing bulk import, split, and compactions from the tablet server.
For now just leave it
- // as a hack instead of trying to make it work correctly with the current
tablet code.
- TabletMetadata tabletMetadata =
- getContext().getAmple().readTablet(getExtent(), ColumnType.FILES);
+ public void refreshMetadata(RefreshPurpose refreshPurpose) {
+ refreshLock.lock();
+ try {
- // TODO this could have race conditions with minor compactions.
Intentionally
- // not being handled ATM.
- getDatafileManager().setFilesHack(tabletMetadata.getFilesMap());
+ // do not want to hold tablet lock while doing metadata read as this
could negatively impact
+ // scans
+ TabletMetadata tabletMetadata =
getContext().getAmple().readTablet(getExtent());
- // ELASTICITY_TODO this was in the code that brought a major compaction
online. Adding it here
- // w/o looking into the larger context too much.
- dataSourceDeletions.incrementAndGet();
+ synchronized (this) {
+ latestMetadata = tabletMetadata;
- // ELASTICITY_TODO this was in the code that brought a major compaction
online. Adding it here
- // w/o looking into the larger context too much.
- computeNumEntries();
+ if (refreshPurpose == RefreshPurpose.MINC_COMPLETION) {
+ // Atomically replace the in memory map with the new file. Before
this synch block a scan
+ // starting would see the in memory map. After this synch block it
should see the file in
+ // the tabletMetadata. Scans sync on the tablet also, so they can
not be in this code
+ // block at the same time.
+
+ lastLocation = null;
+ tabletMemory.finishedMinC();
+
+ // the files and in memory map changed, incrementing this will cause
scans to switch data
+ // sources
+ dataSourceDeletions.incrementAndGet();
+
+ // important to call this after updating latestMetadata and
tabletMemory
+ computeNumEntries();
+ } else if
(!latestMetadata.getFilesMap().equals(tabletMetadata.getFilesMap())) {
+
+ // the files changed, incrementing this will cause scans to switch
data sources
+ dataSourceDeletions.incrementAndGet();
+
+ // important to call this after updating latestMetadata
+ computeNumEntries();
+ }
+ }
+ } finally {
+ refreshLock.unlock();
+ }
- if (!scanEntries.isEmpty()) {
- // ELASTICITY_TODO this is a temporary hack. Should not always remove
scan entries added by a
- // compaction, need to check and see if they are actually in use by a
scan. If in use need to
- // add to a set to remove later. Also should use a conditional mutation
to update, did not
- // bother using conditional mutation as this is temporary.
- var tabletMutator = getContext().getAmple().mutateTablet(extent);
- scanEntries.forEach(tabletMutator::deleteScan);
- tabletMutator.mutate();
+ if (refreshPurpose == RefreshPurpose.REFRESH_RPC) {
+ scanfileManager.removeFilesAfterScan(getMetadata().getScans());
}
}
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
index dd23c5a12b..28ad0e0744 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
@@ -27,7 +27,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -115,7 +114,7 @@ public abstract class TabletBase {
public abstract boolean isClosed();
- public abstract SortedMap<StoredTabletFile,DataFileValue> getDatafiles();
+ public abstract Map<StoredTabletFile,DataFileValue> getDatafiles();
public abstract void addToYieldMetric(int i);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
deleted file mode 100644
index 3b16d22d1a..0000000000
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.tserver.tablet;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.client.admin.TabletHostingGoal;
-import org.apache.accumulo.core.metadata.StoredTabletFile;
-import org.apache.accumulo.core.metadata.schema.DataFileValue;
-import org.apache.accumulo.core.metadata.schema.MetadataTime;
-import org.apache.accumulo.core.metadata.schema.TabletMetadata;
-import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
-import org.apache.accumulo.core.tabletserver.log.LogEntry;
-
-/*
- * Basic information needed to create a tablet.
- */
-public class TabletData {
- private MetadataTime time = null;
- private SortedMap<StoredTabletFile,DataFileValue> dataFiles = new
TreeMap<>();
- private List<LogEntry> logEntries = new ArrayList<>();
- private HashSet<StoredTabletFile> scanFiles = new HashSet<>();
- private long flushID = -1;
- private Location lastLocation = null;
- private String directoryName = null;
- private final TabletHostingGoal goal;
-
- // Read tablet data from metadata tables
- public TabletData(TabletMetadata meta) {
-
- this.time = meta.getTime();
- this.flushID = meta.getFlushId().orElse(-1);
- this.directoryName = meta.getDirName();
- this.logEntries.addAll(meta.getLogs());
- scanFiles.addAll(meta.getScans());
-
- if (meta.getLast() != null) {
- this.lastLocation = meta.getLast();
- }
-
- dataFiles.putAll(meta.getFilesMap());
-
- this.goal = meta.getHostingGoal();
- }
-
- public MetadataTime getTime() {
- return time;
- }
-
- public SortedMap<StoredTabletFile,DataFileValue> getDataFiles() {
- return dataFiles;
- }
-
- public List<LogEntry> getLogEntries() {
- return logEntries;
- }
-
- public HashSet<StoredTabletFile> getScanFiles() {
- return scanFiles;
- }
-
- public long getFlushID() {
- return flushID;
- }
-
- public Location getLastLocation() {
- return lastLocation;
- }
-
- public String getDirectoryName() {
- return directoryName;
- }
-
- public TabletHostingGoal getHostingGoal() {
- return goal;
- }
-}
diff --git
a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
index aefaf31813..e49c5b3716 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
@@ -66,7 +66,6 @@ import
org.apache.accumulo.core.tabletingest.thrift.TabletIngestClientService;
import org.apache.accumulo.core.tabletscan.thrift.ActiveScan;
import org.apache.accumulo.core.tabletscan.thrift.TSamplerConfiguration;
import org.apache.accumulo.core.tabletscan.thrift.TabletScanClientService;
-import org.apache.accumulo.core.tabletserver.thrift.TTabletRefresh;
import org.apache.accumulo.core.tabletserver.thrift.TabletServerClientService;
import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
import org.apache.accumulo.core.util.threads.ThreadPools;
@@ -248,7 +247,7 @@ public class NullTserver {
@Override
public List<TKeyExtent> refreshTablets(TInfo tinfo, TCredentials
credentials,
- List<TTabletRefresh> refreshes) throws TException {
+ List<TKeyExtent> refreshes) throws TException {
return List.of();
}
}
