This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 491c0d6c7e Removes running cache from coordinator (#6244)
491c0d6c7e is described below
commit 491c0d6c7e9253f9fbc1c475dcc890c2149a7ef8
Author: Keith Turner <[email protected]>
AuthorDate: Thu Mar 26 11:32:19 2026 -0400
Removes running cache from coordinator (#6244)
---
.../util/compaction/ExternalCompactionUtil.java | 1 -
.../thrift/CompactionCoordinatorService.java | 728 ---------------------
.../core/compaction/thrift/CompactorService.java | 36 +-
.../compaction/thrift/TExternalCompactionMap.java | 471 -------------
core/src/main/thrift/compaction-coordinator.thrift | 11 -
.../server/compaction/CompactionPluginUtils.java | 35 +
.../java/org/apache/accumulo/manager/Manager.java | 6 -
.../coordinator/CompactionCoordinator.java | 228 +------
.../coordinator/commit/PutGcCandidates.java | 2 -
.../coordinator/commit/RefreshTablet.java | 3 -
.../accumulo/manager/fate/FateWorkerEnv.java | 23 -
.../apache/accumulo/manager/tableOps/FateEnv.java | 3 -
.../compaction/CompactionCoordinatorTest.java | 111 ----
.../accumulo/monitor/next/InformationFetcher.java | 10 +
.../accumulo/monitor/next/SystemInformation.java | 15 +
15 files changed, 80 insertions(+), 1603 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index b11285c0e3..149c12e12b 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@ -340,5 +340,4 @@ public class ExternalCompactionUtil {
throw new IllegalStateException("Exception calling cancel compaction for
" + ecid, e);
}
}
-
}
diff --git
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java
index cf40536be8..4f446373ac 100644
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java
+++
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java
@@ -37,8 +37,6 @@ public class CompactionCoordinatorService {
public void
compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String
exceptionClassName, TCompactionState failureState, java.lang.String groupName,
java.lang.String compactor) throws
org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException,
org.apache.accumulo.core.clientImp [...]
- public void
recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId) throws org.apache.thrift.TException;
-
}
public interface AsyncIface {
@@ -51,8 +49,6 @@ public class CompactionCoordinatorService {
public void
compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String
exceptionClassName, TCompactionState failureState, java.lang.String groupName,
java.lang.String compactor, org.apache.thrift.async.AsyncMethodCallback<Void>
resultHandler) throws org.apache.thrift.TException;
- public void
recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException;
-
}
public static class Client extends org.apache.thrift.TServiceClient
implements Iface {
@@ -209,21 +205,6 @@ public class CompactionCoordinatorService {
return;
}
- @Override
- public void
recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId) throws org.apache.thrift.TException
- {
- send_recordCompletion(tinfo, credentials, externalCompactionId);
- }
-
- public void
send_recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId) throws org.apache.thrift.TException
- {
- recordCompletion_args args = new recordCompletion_args();
- args.setTinfo(tinfo);
- args.setCredentials(credentials);
- args.setExternalCompactionId(externalCompactionId);
- sendBaseOneway("recordCompletion", args);
- }
-
}
public static class AsyncClient extends org.apache.thrift.async.TAsyncClient
implements AsyncIface {
public static class Factory implements
org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
@@ -449,47 +430,6 @@ public class CompactionCoordinatorService {
}
}
- @Override
- public void
recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException {
- checkReady();
- recordCompletion_call method_call = new recordCompletion_call(tinfo,
credentials, externalCompactionId, resultHandler, this, ___protocolFactory,
___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class recordCompletion_call extends
org.apache.thrift.async.TAsyncMethodCall<Void> {
- private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo;
- private org.apache.accumulo.core.securityImpl.thrift.TCredentials
credentials;
- private java.lang.String externalCompactionId;
- public
recordCompletion_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler,
org.apache.thrift.async.TAsyncClient client,
org.apache.thrift.protocol.TProtocolFactory protocolFactory,
org.apache.thrift.transport.TNonblockingTransport transport) throws
org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, true);
- this.tinfo = tinfo;
- this.credentials = credentials;
- this.externalCompactionId = externalCompactionId;
- }
-
- @Override
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws
org.apache.thrift.TException {
- prot.writeMessageBegin(new
org.apache.thrift.protocol.TMessage("recordCompletion",
org.apache.thrift.protocol.TMessageType.ONEWAY, 0));
- recordCompletion_args args = new recordCompletion_args();
- args.setTinfo(tinfo);
- args.setCredentials(credentials);
- args.setExternalCompactionId(externalCompactionId);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- @Override
- public Void getResult() throws org.apache.thrift.TException {
- if (getState() !=
org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new java.lang.IllegalStateException("Method call not
finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport =
new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot =
client.getProtocolFactory().getProtocol(memoryTransport);
- return null;
- }
- }
-
}
public static class Processor<I extends Iface> extends
org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
@@ -507,7 +447,6 @@ public class CompactionCoordinatorService {
processMap.put("getCompactionJob", new getCompactionJob());
processMap.put("updateCompactionStatus", new updateCompactionStatus());
processMap.put("compactionFailed", new compactionFailed());
- processMap.put("recordCompletion", new recordCompletion());
return processMap;
}
@@ -647,33 +586,6 @@ public class CompactionCoordinatorService {
}
}
- public static class recordCompletion<I extends Iface> extends
org.apache.thrift.ProcessFunction<I, recordCompletion_args> {
- public recordCompletion() {
- super("recordCompletion");
- }
-
- @Override
- public recordCompletion_args getEmptyArgsInstance() {
- return new recordCompletion_args();
- }
-
- @Override
- protected boolean isOneway() {
- return true;
- }
-
- @Override
- protected boolean rethrowUnhandledExceptions() {
- return false;
- }
-
- @Override
- public org.apache.thrift.TBase getResult(I iface, recordCompletion_args
args) throws org.apache.thrift.TException {
- iface.recordCompletion(args.tinfo, args.credentials,
args.externalCompactionId);
- return null;
- }
- }
-
}
public static class AsyncProcessor<I extends AsyncIface> extends
org.apache.thrift.TBaseAsyncProcessor<I> {
@@ -691,7 +603,6 @@ public class CompactionCoordinatorService {
processMap.put("getCompactionJob", new getCompactionJob());
processMap.put("updateCompactionStatus", new updateCompactionStatus());
processMap.put("compactionFailed", new compactionFailed());
- processMap.put("recordCompletion", new recordCompletion());
return processMap;
}
@@ -992,46 +903,6 @@ public class CompactionCoordinatorService {
}
}
- public static class recordCompletion<I extends AsyncIface> extends
org.apache.thrift.AsyncProcessFunction<I, recordCompletion_args, Void> {
- public recordCompletion() {
- super("recordCompletion");
- }
-
- @Override
- public recordCompletion_args getEmptyArgsInstance() {
- return new recordCompletion_args();
- }
-
- @Override
- public org.apache.thrift.async.AsyncMethodCallback<Void>
getResultHandler(final
org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final
int seqid) {
- final org.apache.thrift.AsyncProcessFunction fcall = this;
- return new org.apache.thrift.async.AsyncMethodCallback<Void>() {
- @Override
- public void onComplete(Void o) {
- }
- @Override
- public void onError(java.lang.Exception e) {
- if (e instanceof org.apache.thrift.transport.TTransportException) {
- _LOGGER.error("TTransportException inside handler", e);
- fb.close();
- } else {
- _LOGGER.error("Exception inside oneway handler", e);
- }
- }
- };
- }
-
- @Override
- protected boolean isOneway() {
- return true;
- }
-
- @Override
- public void start(I iface, recordCompletion_args args,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException {
- iface.recordCompletion(args.tinfo, args.credentials,
args.externalCompactionId,resultHandler);
- }
- }
-
}
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"})
@@ -6884,604 +6755,5 @@ public class CompactionCoordinatorService {
}
}
- @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"})
- public static class recordCompletion_args implements
org.apache.thrift.TBase<recordCompletion_args, recordCompletion_args._Fields>,
java.io.Serializable, Cloneable, Comparable<recordCompletion_args> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new
org.apache.thrift.protocol.TStruct("recordCompletion_args");
-
- private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC =
new org.apache.thrift.protocol.TField("tinfo",
org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField
CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials",
org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField
EXTERNAL_COMPACTION_ID_FIELD_DESC = new
org.apache.thrift.protocol.TField("externalCompactionId",
org.apache.thrift.protocol.TType.STRING, (short)3);
-
- private static final org.apache.thrift.scheme.SchemeFactory
STANDARD_SCHEME_FACTORY = new recordCompletion_argsStandardSchemeFactory();
- private static final org.apache.thrift.scheme.SchemeFactory
TUPLE_SCHEME_FACTORY = new recordCompletion_argsTupleSchemeFactory();
-
- public @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required
- public @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; //
required
- public @org.apache.thrift.annotation.Nullable java.lang.String
externalCompactionId; // required
-
- /** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- TINFO((short)1, "tinfo"),
- CREDENTIALS((short)2, "credentials"),
- EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId");
-
- private static final java.util.Map<java.lang.String, _Fields> byName =
new java.util.HashMap<java.lang.String, _Fields>();
-
- static {
- for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not
found.
- */
- @org.apache.thrift.annotation.Nullable
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // TINFO
- return TINFO;
- case 2: // CREDENTIALS
- return CREDENTIALS;
- case 3: // EXTERNAL_COMPACTION_ID
- return EXTERNAL_COMPACTION_ID;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new
java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- @org.apache.thrift.annotation.Nullable
- public static _Fields findByName(java.lang.String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final java.lang.String _fieldName;
-
- _Fields(short thriftId, java.lang.String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- @Override
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- @Override
- public java.lang.String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final java.util.Map<_Fields,
org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap
= new java.util.EnumMap<_Fields,
org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.TINFO, new
org.apache.thrift.meta_data.FieldMetaData("tinfo",
org.apache.thrift.TFieldRequirementType.DEFAULT,
- new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
org.apache.accumulo.core.clientImpl.thrift.TInfo.class)));
- tmpMap.put(_Fields.CREDENTIALS, new
org.apache.thrift.meta_data.FieldMetaData("credentials",
org.apache.thrift.TFieldRequirementType.DEFAULT,
- new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
org.apache.accumulo.core.securityImpl.thrift.TCredentials.class)));
- tmpMap.put(_Fields.EXTERNAL_COMPACTION_ID, new
org.apache.thrift.meta_data.FieldMetaData("externalCompactionId",
org.apache.thrift.TFieldRequirementType.DEFAULT,
- new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
-
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(recordCompletion_args.class,
metaDataMap);
- }
-
- public recordCompletion_args() {
- }
-
- public recordCompletion_args(
- org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
- org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
- java.lang.String externalCompactionId)
- {
- this();
- this.tinfo = tinfo;
- this.credentials = credentials;
- this.externalCompactionId = externalCompactionId;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public recordCompletion_args(recordCompletion_args other) {
- if (other.isSetTinfo()) {
- this.tinfo = new
org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo);
- }
- if (other.isSetCredentials()) {
- this.credentials = new
org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials);
- }
- if (other.isSetExternalCompactionId()) {
- this.externalCompactionId = other.externalCompactionId;
- }
- }
-
- @Override
- public recordCompletion_args deepCopy() {
- return new recordCompletion_args(this);
- }
-
- @Override
- public void clear() {
- this.tinfo = null;
- this.credentials = null;
- this.externalCompactionId = null;
- }
-
- @org.apache.thrift.annotation.Nullable
- public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() {
- return this.tinfo;
- }
-
- public recordCompletion_args
setTinfo(@org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) {
- this.tinfo = tinfo;
- return this;
- }
-
- public void unsetTinfo() {
- this.tinfo = null;
- }
-
- /** Returns true if field tinfo is set (has been assigned a value) and
false otherwise */
- public boolean isSetTinfo() {
- return this.tinfo != null;
- }
-
- public void setTinfoIsSet(boolean value) {
- if (!value) {
- this.tinfo = null;
- }
- }
-
- @org.apache.thrift.annotation.Nullable
- public org.apache.accumulo.core.securityImpl.thrift.TCredentials
getCredentials() {
- return this.credentials;
- }
-
- public recordCompletion_args
setCredentials(@org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) {
- this.credentials = credentials;
- return this;
- }
-
- public void unsetCredentials() {
- this.credentials = null;
- }
-
- /** Returns true if field credentials is set (has been assigned a value)
and false otherwise */
- public boolean isSetCredentials() {
- return this.credentials != null;
- }
-
- public void setCredentialsIsSet(boolean value) {
- if (!value) {
- this.credentials = null;
- }
- }
-
- @org.apache.thrift.annotation.Nullable
- public java.lang.String getExternalCompactionId() {
- return this.externalCompactionId;
- }
-
- public recordCompletion_args
setExternalCompactionId(@org.apache.thrift.annotation.Nullable java.lang.String
externalCompactionId) {
- this.externalCompactionId = externalCompactionId;
- return this;
- }
-
- public void unsetExternalCompactionId() {
- this.externalCompactionId = null;
- }
-
- /** Returns true if field externalCompactionId is set (has been assigned a
value) and false otherwise */
- public boolean isSetExternalCompactionId() {
- return this.externalCompactionId != null;
- }
-
- public void setExternalCompactionIdIsSet(boolean value) {
- if (!value) {
- this.externalCompactionId = null;
- }
- }
-
- @Override
- public void setFieldValue(_Fields field,
@org.apache.thrift.annotation.Nullable java.lang.Object value) {
- switch (field) {
- case TINFO:
- if (value == null) {
- unsetTinfo();
- } else {
- setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value);
- }
- break;
-
- case CREDENTIALS:
- if (value == null) {
- unsetCredentials();
- } else {
-
setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value);
- }
- break;
-
- case EXTERNAL_COMPACTION_ID:
- if (value == null) {
- unsetExternalCompactionId();
- } else {
- setExternalCompactionId((java.lang.String)value);
- }
- break;
-
- }
- }
-
- @org.apache.thrift.annotation.Nullable
- @Override
- public java.lang.Object getFieldValue(_Fields field) {
- switch (field) {
- case TINFO:
- return getTinfo();
-
- case CREDENTIALS:
- return getCredentials();
-
- case EXTERNAL_COMPACTION_ID:
- return getExternalCompactionId();
-
- }
- throw new java.lang.IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been
assigned a value) and false otherwise */
- @Override
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new java.lang.IllegalArgumentException();
- }
-
- switch (field) {
- case TINFO:
- return isSetTinfo();
- case CREDENTIALS:
- return isSetCredentials();
- case EXTERNAL_COMPACTION_ID:
- return isSetExternalCompactionId();
- }
- throw new java.lang.IllegalStateException();
- }
-
- @Override
- public boolean equals(java.lang.Object that) {
- if (that instanceof recordCompletion_args)
- return this.equals((recordCompletion_args)that);
- return false;
- }
-
- public boolean equals(recordCompletion_args that) {
- if (that == null)
- return false;
- if (this == that)
- return true;
-
- boolean this_present_tinfo = true && this.isSetTinfo();
- boolean that_present_tinfo = true && that.isSetTinfo();
- if (this_present_tinfo || that_present_tinfo) {
- if (!(this_present_tinfo && that_present_tinfo))
- return false;
- if (!this.tinfo.equals(that.tinfo))
- return false;
- }
-
- boolean this_present_credentials = true && this.isSetCredentials();
- boolean that_present_credentials = true && that.isSetCredentials();
- if (this_present_credentials || that_present_credentials) {
- if (!(this_present_credentials && that_present_credentials))
- return false;
- if (!this.credentials.equals(that.credentials))
- return false;
- }
-
- boolean this_present_externalCompactionId = true &&
this.isSetExternalCompactionId();
- boolean that_present_externalCompactionId = true &&
that.isSetExternalCompactionId();
- if (this_present_externalCompactionId ||
that_present_externalCompactionId) {
- if (!(this_present_externalCompactionId &&
that_present_externalCompactionId))
- return false;
- if (!this.externalCompactionId.equals(that.externalCompactionId))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int hashCode = 1;
-
- hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287);
- if (isSetTinfo())
- hashCode = hashCode * 8191 + tinfo.hashCode();
-
- hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287);
- if (isSetCredentials())
- hashCode = hashCode * 8191 + credentials.hashCode();
-
- hashCode = hashCode * 8191 + ((isSetExternalCompactionId()) ? 131071 :
524287);
- if (isSetExternalCompactionId())
- hashCode = hashCode * 8191 + externalCompactionId.hashCode();
-
- return hashCode;
- }
-
- @Override
- public int compareTo(recordCompletion_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = java.lang.Boolean.compare(isSetTinfo(),
other.isSetTinfo());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTinfo()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo,
other.tinfo);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = java.lang.Boolean.compare(isSetCredentials(),
other.isSetCredentials());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCredentials()) {
- lastComparison =
org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = java.lang.Boolean.compare(isSetExternalCompactionId(),
other.isSetExternalCompactionId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetExternalCompactionId()) {
- lastComparison =
org.apache.thrift.TBaseHelper.compareTo(this.externalCompactionId,
other.externalCompactionId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- @org.apache.thrift.annotation.Nullable
- @Override
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws
org.apache.thrift.TException {
- scheme(iprot).read(iprot, this);
- }
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws
org.apache.thrift.TException {
- scheme(oprot).write(oprot, this);
- }
-
- @Override
- public java.lang.String toString() {
- java.lang.StringBuilder sb = new
java.lang.StringBuilder("recordCompletion_args(");
- boolean first = true;
-
- sb.append("tinfo:");
- if (this.tinfo == null) {
- sb.append("null");
- } else {
- sb.append(this.tinfo);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("credentials:");
- if (this.credentials == null) {
- sb.append("null");
- } else {
- sb.append(this.credentials);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("externalCompactionId:");
- if (this.externalCompactionId == null) {
- sb.append("null");
- } else {
- sb.append(this.externalCompactionId);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (tinfo != null) {
- tinfo.validate();
- }
- if (credentials != null) {
- credentials.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws
java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new
org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws
java.io.IOException, java.lang.ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new
org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class recordCompletion_argsStandardSchemeFactory implements
org.apache.thrift.scheme.SchemeFactory {
- @Override
- public recordCompletion_argsStandardScheme getScheme() {
- return new recordCompletion_argsStandardScheme();
- }
- }
-
- private static class recordCompletion_argsStandardScheme extends
org.apache.thrift.scheme.StandardScheme<recordCompletion_args> {
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol iprot,
recordCompletion_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // TINFO
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT)
{
- struct.tinfo = new
org.apache.accumulo.core.clientImpl.thrift.TInfo();
- struct.tinfo.read(iprot);
- struct.setTinfoIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- break;
- case 2: // CREDENTIALS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT)
{
- struct.credentials = new
org.apache.accumulo.core.securityImpl.thrift.TCredentials();
- struct.credentials.read(iprot);
- struct.setCredentialsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- break;
- case 3: // EXTERNAL_COMPACTION_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING)
{
- struct.externalCompactionId = iprot.readString();
- struct.setExternalCompactionIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked
in the validate method
- struct.validate();
- }
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol oprot,
recordCompletion_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.tinfo != null) {
- oprot.writeFieldBegin(TINFO_FIELD_DESC);
- struct.tinfo.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.credentials != null) {
- oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC);
- struct.credentials.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.externalCompactionId != null) {
- oprot.writeFieldBegin(EXTERNAL_COMPACTION_ID_FIELD_DESC);
- oprot.writeString(struct.externalCompactionId);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class recordCompletion_argsTupleSchemeFactory implements
org.apache.thrift.scheme.SchemeFactory {
- @Override
- public recordCompletion_argsTupleScheme getScheme() {
- return new recordCompletion_argsTupleScheme();
- }
- }
-
- private static class recordCompletion_argsTupleScheme extends
org.apache.thrift.scheme.TupleScheme<recordCompletion_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot,
recordCompletion_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TTupleProtocol oprot =
(org.apache.thrift.protocol.TTupleProtocol) prot;
- java.util.BitSet optionals = new java.util.BitSet();
- if (struct.isSetTinfo()) {
- optionals.set(0);
- }
- if (struct.isSetCredentials()) {
- optionals.set(1);
- }
- if (struct.isSetExternalCompactionId()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetTinfo()) {
- struct.tinfo.write(oprot);
- }
- if (struct.isSetCredentials()) {
- struct.credentials.write(oprot);
- }
- if (struct.isSetExternalCompactionId()) {
- oprot.writeString(struct.externalCompactionId);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot,
recordCompletion_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TTupleProtocol iprot =
(org.apache.thrift.protocol.TTupleProtocol) prot;
- java.util.BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.tinfo = new
org.apache.accumulo.core.clientImpl.thrift.TInfo();
- struct.tinfo.read(iprot);
- struct.setTinfoIsSet(true);
- }
- if (incoming.get(1)) {
- struct.credentials = new
org.apache.accumulo.core.securityImpl.thrift.TCredentials();
- struct.credentials.read(iprot);
- struct.setCredentialsIsSet(true);
- }
- if (incoming.get(2)) {
- struct.externalCompactionId = iprot.readString();
- struct.setExternalCompactionIdIsSet(true);
- }
- }
- }
-
- private static <S extends org.apache.thrift.scheme.IScheme> S
scheme(org.apache.thrift.protocol.TProtocol proto) {
- return
(org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ?
STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme();
- }
- }
-
private static void unusedMethod() {}
}
diff --git
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java
index d9d867d8d2..c59d19d98f 100644
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java
+++
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java
@@ -3668,14 +3668,14 @@ public class CompactorService {
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list28 =
iprot.readListBegin();
- struct.success = new
java.util.ArrayList<org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction>(_list28.size);
- @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem29;
- for (int _i30 = 0; _i30 < _list28.size; ++_i30)
+ org.apache.thrift.protocol.TList _list18 =
iprot.readListBegin();
+ struct.success = new
java.util.ArrayList<org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction>(_list18.size);
+ @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem19;
+ for (int _i20 = 0; _i20 < _list18.size; ++_i20)
{
- _elem29 = new
org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction();
- _elem29.read(iprot);
- struct.success.add(_elem29);
+ _elem19 = new
org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction();
+ _elem19.read(iprot);
+ struct.success.add(_elem19);
}
iprot.readListEnd();
}
@@ -3713,9 +3713,9 @@ public class CompactorService {
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.success.size()));
- for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction
_iter31 : struct.success)
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction
_iter21 : struct.success)
{
- _iter31.write(oprot);
+ _iter21.write(oprot);
}
oprot.writeListEnd();
}
@@ -3755,9 +3755,9 @@ public class CompactorService {
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
- for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction
_iter32 : struct.success)
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction
_iter22 : struct.success)
{
- _iter32.write(oprot);
+ _iter22.write(oprot);
}
}
}
@@ -3772,14 +3772,14 @@ public class CompactorService {
java.util.BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list33 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
- struct.success = new
java.util.ArrayList<org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction>(_list33.size);
- @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem34;
- for (int _i35 = 0; _i35 < _list33.size; ++_i35)
+ org.apache.thrift.protocol.TList _list23 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT);
+ struct.success = new
java.util.ArrayList<org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction>(_list23.size);
+ @org.apache.thrift.annotation.Nullable
org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem24;
+ for (int _i25 = 0; _i25 < _list23.size; ++_i25)
{
- _elem34 = new
org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction();
- _elem34.read(iprot);
- struct.success.add(_elem34);
+ _elem24 = new
org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction();
+ _elem24.read(iprot);
+ struct.success.add(_elem24);
}
}
struct.setSuccessIsSet(true);
diff --git
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java
deleted file mode 100644
index a5b3d74a2b..0000000000
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java
+++ /dev/null
@@ -1,471 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Autogenerated by Thrift Compiler (0.17.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.accumulo.core.compaction.thrift;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"})
-public class TExternalCompactionMap implements
org.apache.thrift.TBase<TExternalCompactionMap,
TExternalCompactionMap._Fields>, java.io.Serializable, Cloneable,
Comparable<TExternalCompactionMap> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new
org.apache.thrift.protocol.TStruct("TExternalCompactionMap");
-
- private static final org.apache.thrift.protocol.TField
COMPACTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compactions",
org.apache.thrift.protocol.TType.MAP, (short)1);
-
- private static final org.apache.thrift.scheme.SchemeFactory
STANDARD_SCHEME_FACTORY = new TExternalCompactionMapStandardSchemeFactory();
- private static final org.apache.thrift.scheme.SchemeFactory
TUPLE_SCHEME_FACTORY = new TExternalCompactionMapTupleSchemeFactory();
-
- public @org.apache.thrift.annotation.Nullable
java.util.Map<java.lang.String,TExternalCompaction> compactions; // required
-
- /** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- COMPACTIONS((short)1, "compactions");
-
- private static final java.util.Map<java.lang.String, _Fields> byName = new
java.util.HashMap<java.lang.String, _Fields>();
-
- static {
- for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not
found.
- */
- @org.apache.thrift.annotation.Nullable
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // COMPACTIONS
- return COMPACTIONS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new java.lang.IllegalArgumentException("Field
" + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- @org.apache.thrift.annotation.Nullable
- public static _Fields findByName(java.lang.String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final java.lang.String _fieldName;
-
- _Fields(short thriftId, java.lang.String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- @Override
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- @Override
- public java.lang.String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final java.util.Map<_Fields,
org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap =
new java.util.EnumMap<_Fields,
org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.COMPACTIONS, new
org.apache.thrift.meta_data.FieldMetaData("compactions",
org.apache.thrift.TFieldRequirementType.DEFAULT,
- new
org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
TExternalCompaction.class))));
- metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
-
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TExternalCompactionMap.class,
metaDataMap);
- }
-
- public TExternalCompactionMap() {
- }
-
- public TExternalCompactionMap(
- java.util.Map<java.lang.String,TExternalCompaction> compactions)
- {
- this();
- this.compactions = compactions;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TExternalCompactionMap(TExternalCompactionMap other) {
- if (other.isSetCompactions()) {
- java.util.Map<java.lang.String,TExternalCompaction> __this__compactions
= new
java.util.HashMap<java.lang.String,TExternalCompaction>(other.compactions.size());
- for (java.util.Map.Entry<java.lang.String, TExternalCompaction>
other_element : other.compactions.entrySet()) {
-
- java.lang.String other_element_key = other_element.getKey();
- TExternalCompaction other_element_value = other_element.getValue();
-
- java.lang.String __this__compactions_copy_key = other_element_key;
-
- TExternalCompaction __this__compactions_copy_value = new
TExternalCompaction(other_element_value);
-
- __this__compactions.put(__this__compactions_copy_key,
__this__compactions_copy_value);
- }
- this.compactions = __this__compactions;
- }
- }
-
- @Override
- public TExternalCompactionMap deepCopy() {
- return new TExternalCompactionMap(this);
- }
-
- @Override
- public void clear() {
- this.compactions = null;
- }
-
- public int getCompactionsSize() {
- return (this.compactions == null) ? 0 : this.compactions.size();
- }
-
- public void putToCompactions(java.lang.String key, TExternalCompaction val) {
- if (this.compactions == null) {
- this.compactions = new
java.util.HashMap<java.lang.String,TExternalCompaction>();
- }
- this.compactions.put(key, val);
- }
-
- @org.apache.thrift.annotation.Nullable
- public java.util.Map<java.lang.String,TExternalCompaction> getCompactions() {
- return this.compactions;
- }
-
- public TExternalCompactionMap
setCompactions(@org.apache.thrift.annotation.Nullable
java.util.Map<java.lang.String,TExternalCompaction> compactions) {
- this.compactions = compactions;
- return this;
- }
-
- public void unsetCompactions() {
- this.compactions = null;
- }
-
- /** Returns true if field compactions is set (has been assigned a value) and
false otherwise */
- public boolean isSetCompactions() {
- return this.compactions != null;
- }
-
- public void setCompactionsIsSet(boolean value) {
- if (!value) {
- this.compactions = null;
- }
- }
-
- @Override
- public void setFieldValue(_Fields field,
@org.apache.thrift.annotation.Nullable java.lang.Object value) {
- switch (field) {
- case COMPACTIONS:
- if (value == null) {
- unsetCompactions();
- } else {
-
setCompactions((java.util.Map<java.lang.String,TExternalCompaction>)value);
- }
- break;
-
- }
- }
-
- @org.apache.thrift.annotation.Nullable
- @Override
- public java.lang.Object getFieldValue(_Fields field) {
- switch (field) {
- case COMPACTIONS:
- return getCompactions();
-
- }
- throw new java.lang.IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned
a value) and false otherwise */
- @Override
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new java.lang.IllegalArgumentException();
- }
-
- switch (field) {
- case COMPACTIONS:
- return isSetCompactions();
- }
- throw new java.lang.IllegalStateException();
- }
-
- @Override
- public boolean equals(java.lang.Object that) {
- if (that instanceof TExternalCompactionMap)
- return this.equals((TExternalCompactionMap)that);
- return false;
- }
-
- public boolean equals(TExternalCompactionMap that) {
- if (that == null)
- return false;
- if (this == that)
- return true;
-
- boolean this_present_compactions = true && this.isSetCompactions();
- boolean that_present_compactions = true && that.isSetCompactions();
- if (this_present_compactions || that_present_compactions) {
- if (!(this_present_compactions && that_present_compactions))
- return false;
- if (!this.compactions.equals(that.compactions))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int hashCode = 1;
-
- hashCode = hashCode * 8191 + ((isSetCompactions()) ? 131071 : 524287);
- if (isSetCompactions())
- hashCode = hashCode * 8191 + compactions.hashCode();
-
- return hashCode;
- }
-
- @Override
- public int compareTo(TExternalCompactionMap other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
-
- lastComparison = java.lang.Boolean.compare(isSetCompactions(),
other.isSetCompactions());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCompactions()) {
- lastComparison =
org.apache.thrift.TBaseHelper.compareTo(this.compactions, other.compactions);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- @org.apache.thrift.annotation.Nullable
- @Override
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws
org.apache.thrift.TException {
- scheme(iprot).read(iprot, this);
- }
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws
org.apache.thrift.TException {
- scheme(oprot).write(oprot, this);
- }
-
- @Override
- public java.lang.String toString() {
- java.lang.StringBuilder sb = new
java.lang.StringBuilder("TExternalCompactionMap(");
- boolean first = true;
-
- sb.append("compactions:");
- if (this.compactions == null) {
- sb.append("null");
- } else {
- sb.append(this.compactions);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws
java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new
org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws
java.io.IOException, java.lang.ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new
org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TExternalCompactionMapStandardSchemeFactory implements
org.apache.thrift.scheme.SchemeFactory {
- @Override
- public TExternalCompactionMapStandardScheme getScheme() {
- return new TExternalCompactionMapStandardScheme();
- }
- }
-
- private static class TExternalCompactionMapStandardScheme extends
org.apache.thrift.scheme.StandardScheme<TExternalCompactionMap> {
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol iprot,
TExternalCompactionMap struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // COMPACTIONS
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map18 = iprot.readMapBegin();
- struct.compactions = new
java.util.HashMap<java.lang.String,TExternalCompaction>(2*_map18.size);
- @org.apache.thrift.annotation.Nullable java.lang.String _key19;
- @org.apache.thrift.annotation.Nullable TExternalCompaction
_val20;
- for (int _i21 = 0; _i21 < _map18.size; ++_i21)
- {
- _key19 = iprot.readString();
- _val20 = new TExternalCompaction();
- _val20.read(iprot);
- struct.compactions.put(_key19, _val20);
- }
- iprot.readMapEnd();
- }
- struct.setCompactionsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked
in the validate method
- struct.validate();
- }
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol oprot,
TExternalCompactionMap struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.compactions != null) {
- oprot.writeFieldBegin(COMPACTIONS_FIELD_DESC);
- {
- oprot.writeMapBegin(new
org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.STRUCT, struct.compactions.size()));
- for (java.util.Map.Entry<java.lang.String, TExternalCompaction>
_iter22 : struct.compactions.entrySet())
- {
- oprot.writeString(_iter22.getKey());
- _iter22.getValue().write(oprot);
- }
- oprot.writeMapEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TExternalCompactionMapTupleSchemeFactory implements
org.apache.thrift.scheme.SchemeFactory {
- @Override
- public TExternalCompactionMapTupleScheme getScheme() {
- return new TExternalCompactionMapTupleScheme();
- }
- }
-
- private static class TExternalCompactionMapTupleScheme extends
org.apache.thrift.scheme.TupleScheme<TExternalCompactionMap> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot,
TExternalCompactionMap struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TTupleProtocol oprot =
(org.apache.thrift.protocol.TTupleProtocol) prot;
- java.util.BitSet optionals = new java.util.BitSet();
- if (struct.isSetCompactions()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetCompactions()) {
- {
- oprot.writeI32(struct.compactions.size());
- for (java.util.Map.Entry<java.lang.String, TExternalCompaction>
_iter23 : struct.compactions.entrySet())
- {
- oprot.writeString(_iter23.getKey());
- _iter23.getValue().write(oprot);
- }
- }
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot,
TExternalCompactionMap struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TTupleProtocol iprot =
(org.apache.thrift.protocol.TTupleProtocol) prot;
- java.util.BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- {
- org.apache.thrift.protocol.TMap _map24 =
iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.STRUCT);
- struct.compactions = new
java.util.HashMap<java.lang.String,TExternalCompaction>(2*_map24.size);
- @org.apache.thrift.annotation.Nullable java.lang.String _key25;
- @org.apache.thrift.annotation.Nullable TExternalCompaction _val26;
- for (int _i27 = 0; _i27 < _map24.size; ++_i27)
- {
- _key25 = iprot.readString();
- _val26 = new TExternalCompaction();
- _val26.read(iprot);
- struct.compactions.put(_key25, _val26);
- }
- }
- struct.setCompactionsIsSet(true);
- }
- }
- }
-
- private static <S extends org.apache.thrift.scheme.IScheme> S
scheme(org.apache.thrift.protocol.TProtocol proto) {
- return
(org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ?
STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme();
- }
- private static void unusedMethod() {}
-}
-
diff --git a/core/src/main/thrift/compaction-coordinator.thrift
b/core/src/main/thrift/compaction-coordinator.thrift
index 29e4dde868..8effd88aff 100644
--- a/core/src/main/thrift/compaction-coordinator.thrift
+++ b/core/src/main/thrift/compaction-coordinator.thrift
@@ -60,10 +60,6 @@ struct TExternalCompactionList {
1:list<TExternalCompaction> compactions
}
-struct TExternalCompactionMap {
- 1:map<string,TExternalCompaction> compactions
-}
-
struct TNextCompactionJob {
1:tabletserver.TExternalCompactionJob job
// The total number of compactors servicing the queue this job was requested
for
@@ -135,13 +131,6 @@ service CompactionCoordinatorService {
1:client.ThriftSecurityException sec
2:client.ThriftNotActiveServiceException tnase
)
-
-
- oneway void recordCompletion(
- 1:client.TInfo tinfo
- 2:security.TCredentials credentials
- 3:string externalCompactionId
- )
}
service CompactorService {
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionPluginUtils.java
b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionPluginUtils.java
index 854f096ebd..514a110f2a 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionPluginUtils.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionPluginUtils.java
@@ -24,6 +24,7 @@ import java.net.URI;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -46,6 +47,7 @@ import
org.apache.accumulo.core.clientImpl.UserCompactionUtils;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.AbstractId;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.RowRange;
import org.apache.accumulo.core.data.TableId;
@@ -62,9 +64,13 @@ import
org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
import org.apache.accumulo.core.spi.compaction.CompactionDispatcher;
+import org.apache.accumulo.core.spi.compaction.CompactionPlanner;
+import org.apache.accumulo.core.spi.compaction.CompactionServiceId;
import org.apache.accumulo.core.summary.SummarizerFactory;
import org.apache.accumulo.core.summary.SummaryCollection;
import org.apache.accumulo.core.summary.SummaryReader;
+import org.apache.accumulo.core.util.compaction.CompactionPlannerInitParams;
+import org.apache.accumulo.core.util.compaction.CompactionServicesConfig;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.ServiceEnvironmentImpl;
import org.apache.accumulo.server.tablets.TabletNameGenerator;
@@ -345,4 +351,33 @@ public class CompactionPluginUtils {
return dispatcher;
}
+
+ /**
+ * Inspect configuration and determines what resource groups are configured
for compaction.
+ */
+ public static Set<String>
getConfiguredCompactionResourceGroups(ServerContext ctx)
+ throws ReflectiveOperationException {
+
+ Set<String> groups = new HashSet<>();
+ AccumuloConfiguration config = ctx.getConfiguration();
+ CompactionServicesConfig servicesConfig = new
CompactionServicesConfig(config);
+
+ for (var entry : servicesConfig.getPlanners().entrySet()) {
+ String serviceId = entry.getKey();
+ String plannerClassName = entry.getValue();
+
+ Class<? extends CompactionPlanner> plannerClass =
+ Class.forName(plannerClassName).asSubclass(CompactionPlanner.class);
+ CompactionPlanner planner =
plannerClass.getDeclaredConstructor().newInstance();
+
+ var initParams = new
CompactionPlannerInitParams(CompactionServiceId.of(serviceId),
+ servicesConfig.getPlannerPrefix(serviceId),
servicesConfig.getOptions().get(serviceId),
+ new ServiceEnvironmentImpl(ctx));
+
+ planner.init(initParams);
+
+
initParams.getRequestedGroups().stream().map(AbstractId::canonical).forEach(groups::add);
+ }
+ return groups;
+ }
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 5ff232765c..60e09f3dd8 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@ -107,7 +107,6 @@ import
org.apache.accumulo.core.manager.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.SystemTables;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
-import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.metrics.MetricsProducer;
import org.apache.accumulo.core.trace.TraceUtil;
@@ -609,11 +608,6 @@ public class Manager extends AbstractServer
return compactionCoordinator;
}
- @Override
- public void recordCompactionCompletion(ExternalCompactionId ecid) {
- getCompactionCoordinator().recordCompletion(ecid);
- }
-
public void hostOndemand(List<KeyExtent> extents) {
extents.forEach(e -> Preconditions.checkArgument(DataLevel.of(e.tableId())
== DataLevel.USER));
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index 0ff67d4850..4da7b76fa5 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -27,7 +27,6 @@ import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
-import static
org.apache.accumulo.core.util.threads.ThreadPoolNames.COMPACTOR_RUNNING_COMPACTIONS_POOL;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -62,16 +61,13 @@ import
org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableDeletedException;
import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
-import org.apache.accumulo.core.client.admin.servers.ServerId;
import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode;
import org.apache.accumulo.core.clientImpl.thrift.TInfo;
import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService;
import org.apache.accumulo.core.compaction.thrift.TCompactionState;
import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate;
-import org.apache.accumulo.core.compaction.thrift.TExternalCompaction;
import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.ResourceGroupId;
import org.apache.accumulo.core.data.TableId;
@@ -104,16 +100,12 @@ import org.apache.accumulo.core.metrics.MetricsProducer;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
-import org.apache.accumulo.core.spi.compaction.CompactionPlanner;
-import org.apache.accumulo.core.spi.compaction.CompactionServiceId;
import org.apache.accumulo.core.tabletserver.thrift.InputFile;
import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
import org.apache.accumulo.core.tabletserver.thrift.TCompactionKind;
import org.apache.accumulo.core.tabletserver.thrift.TCompactionStats;
import org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob;
import org.apache.accumulo.core.util.cache.Caches.CacheName;
-import org.apache.accumulo.core.util.compaction.CompactionPlannerInitParams;
-import org.apache.accumulo.core.util.compaction.CompactionServicesConfig;
import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.core.util.threads.Threads;
@@ -127,7 +119,6 @@ import
org.apache.accumulo.manager.compaction.queue.CompactionJobQueues;
import org.apache.accumulo.manager.compaction.queue.ResolvedCompactionJob;
import org.apache.accumulo.manager.tableOps.FateEnv;
import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.ServiceEnvironmentImpl;
import org.apache.accumulo.server.compaction.CompactionConfigStorage;
import org.apache.accumulo.server.compaction.CompactionPluginUtils;
import org.apache.accumulo.server.security.AuditedSecurityOperation;
@@ -135,7 +126,6 @@ import
org.apache.accumulo.server.tablets.TabletNameGenerator;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.thrift.TException;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -146,7 +136,6 @@ import com.github.benmanes.caffeine.cache.LoadingCache;
import com.github.benmanes.caffeine.cache.Weigher;
import com.google.common.base.Preconditions;
import com.google.common.base.Suppliers;
-import com.google.common.collect.Sets;
import com.google.common.net.HostAndPort;
import io.micrometer.core.instrument.MeterRegistry;
@@ -191,16 +180,6 @@ public class CompactionCoordinator
public static final String RESTART_UPDATE_MSG =
"Coordinator restarted, compaction found in progress";
- /*
- * Map of compactionId to TExternalCompaction. This is an informational
cache of what external
- * compactions may be running. Its possible it may contain external
compactions that are not
- * actually running. It may not contain compactions that are actually
running. The metadata table
- * is the most authoritative source of what external compactions are
currently running, but it
- * does not have the stats that this map has.
- */
- protected final Map<ExternalCompactionId,TExternalCompaction> RUNNING_CACHE =
- new ConcurrentHashMap<>();
-
private final ServerContext ctx;
private final AuditedSecurityOperation security;
private final CompactionJobQueues jobQueues;
@@ -308,7 +287,7 @@ public class CompactionCoordinator
protected void startInternalStateCleaner(ScheduledThreadPoolExecutor
schedExecutor) {
ScheduledFuture<?> future =
- schedExecutor.scheduleWithFixedDelay(this::cleanUpInternalState, 0, 5,
TimeUnit.MINUTES);
+ schedExecutor.scheduleWithFixedDelay(this::resizeThreadPools, 0, 5,
TimeUnit.MINUTES);
ThreadPools.watchNonCriticalScheduledTask(future);
}
@@ -331,30 +310,6 @@ public class CompactionCoordinator
startConfigMonitor(ctx.getScheduledExecutor());
startCompactorZKCleaner(ctx.getScheduledExecutor());
- // On a re-start of the coordinator it's possible that external
compactions are in-progress.
- // Attempt to get the running compactions on the compactors and then
resolve which tserver
- // the external compaction came from to re-populate the RUNNING collection.
- LOG.info("Checking for running external compactions");
- // On re-start contact the running Compactors to try and seed the list of
running compactions
- try {
- List<TExternalCompaction> running = getCompactionsRunningOnCompactors();
- if (running.isEmpty()) {
- LOG.info("No running external compactions found");
- } else {
- LOG.info("Found {} running external compactions", running.size());
- running.forEach(tec -> {
- TCompactionStatusUpdate update = new TCompactionStatusUpdate();
- update.setState(TCompactionState.IN_PROGRESS);
- update.setMessage(RESTART_UPDATE_MSG);
- tec.putToUpdates(coordinatorStartTime, update);
-
RUNNING_CACHE.put(ExternalCompactionId.of(tec.getJob().getExternalCompactionId()),
tec);
- });
- }
- } catch (InterruptedException e) {
- throw new IllegalStateException(
- "Thread interrupted while retrieving running compactions from
compactors", e);
- }
-
startDeadCompactionDetector();
startFailureSummaryLogging();
startInternalStateCleaner(ctx.getScheduledExecutor());
@@ -372,14 +327,6 @@ public class CompactionCoordinator
deadCompactionDetector.start();
}
- protected long getMissingCompactorWarningTime() {
- return
this.ctx.getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME)
* 3;
- }
-
- public long getNumRunningCompactions() {
- return RUNNING_CACHE.size();
- }
-
/**
* Return the next compaction job from the queue to a Compactor
*
@@ -425,13 +372,6 @@ public class CompactionCoordinator
if (ecm != null) {
result = createThriftJob(externalCompactionId, ecm, rcJob,
compactionConfig);
- // It is possible that by the time this added that the the compactor
that made this request
- // is dead. In this cases the compaction is not actually running.
- TExternalCompaction tec = new TExternalCompaction();
- tec.setCompactor(compactorAddress);
- tec.setGroupName(groupName);
- tec.setJob(result);
-
RUNNING_CACHE.put(ExternalCompactionId.of(result.getExternalCompactionId()),
tec);
TabletLogger.compacting(rcJob.getExtent(), rcJob.getSelectedFateId(),
cid, compactorAddress,
rcJob, ecm.getCompactTmpName());
break;
@@ -913,8 +853,6 @@ public class CompactionCoordinator
}
});
}
-
- compactions.values().forEach(ecids ->
ecids.forEach(this::recordCompletion));
}
/**
@@ -938,26 +876,6 @@ public class CompactionCoordinator
}
LOG.debug("Compaction status update, id: {}, timestamp: {}, update: {}",
externalCompactionId,
timestamp, update);
- final TExternalCompaction tec =
- RUNNING_CACHE.get(ExternalCompactionId.of(externalCompactionId));
- if (null != tec) {
- if (update.getState() == TCompactionState.STARTED) {
- tec.setStartTime(timestamp);
- }
- tec.putToUpdates(timestamp, update);
- }
- }
-
- @Override
- public void recordCompletion(TInfo tinfo, TCredentials credentials, String
externalCompactionId)
- throws TException {
- if (security.canPerformSystemActions(credentials)) {
- recordCompletion(ExternalCompactionId.of(externalCompactionId));
- }
- }
-
- public void recordCompletion(ExternalCompactionId ecid) {
- var tec = RUNNING_CACHE.remove(ecid);
}
protected Set<ExternalCompactionId> readExternalCompactionIds() {
@@ -974,35 +892,6 @@ public class CompactionCoordinator
return jobQueues;
}
- /* Method exists to be overridden in test to hide static method */
- protected List<TExternalCompaction> getCompactionsRunningOnCompactors()
- throws InterruptedException {
- int numCompactors =
this.ctx.instanceOperations().getServers(ServerId.Type.COMPACTOR).size();
- final ExecutorService executor =
-
ThreadPools.getServerThreadPools().getPoolBuilder(COMPACTOR_RUNNING_COMPACTIONS_POOL)
- .numCoreThreads(numCompactors / 10).build();
- try {
- List<TExternalCompaction> running = new ArrayList<>();
- @SuppressWarnings("unused")
- List<ServerId> failures =
ExternalCompactionUtil.getCompactionsRunningOnCompactors(this.ctx,
- executor, (t) -> running.add(t));
- return running;
- } finally {
- executor.shutdownNow();
- }
- }
-
- /* Method exists to be overridden in test to hide static method */
- protected Set<ServerId> getRunningCompactors() {
- return ctx.instanceOperations().getServers(ServerId.Type.COMPACTOR);
- }
-
- /* Method exists to be overridden in test to hide static method */
- protected void cancelCompactionOnCompactor(String address, String
externalCompactionId) {
- HostAndPort hostPort = HostAndPort.fromString(address);
- ExternalCompactionUtil.cancelCompaction(this.ctx, hostPort,
externalCompactionId);
- }
-
private void deleteEmpty(ZooReaderWriter zoorw, String path)
throws KeeperException, InterruptedException {
try {
@@ -1052,44 +941,7 @@ public class CompactionCoordinator
}
}
- private Set<ResourceGroupId> getCompactionServicesConfigurationGroups()
- throws ReflectiveOperationException, IllegalArgumentException,
SecurityException {
-
- Set<ResourceGroupId> groups = new HashSet<>();
- AccumuloConfiguration config = ctx.getConfiguration();
- CompactionServicesConfig servicesConfig = new
CompactionServicesConfig(config);
-
- for (var entry : servicesConfig.getPlanners().entrySet()) {
- String serviceId = entry.getKey();
- String plannerClassName = entry.getValue();
-
- Class<? extends CompactionPlanner> plannerClass =
- Class.forName(plannerClassName).asSubclass(CompactionPlanner.class);
- CompactionPlanner planner =
plannerClass.getDeclaredConstructor().newInstance();
-
- var initParams = new
CompactionPlannerInitParams(CompactionServiceId.of(serviceId),
- servicesConfig.getPlannerPrefix(serviceId),
servicesConfig.getOptions().get(serviceId),
- new ServiceEnvironmentImpl(ctx));
-
- planner.init(initParams);
-
- groups.addAll(initParams.getRequestedGroups());
- }
- return groups;
- }
-
- public void cleanUpInternalState() {
-
- // This method does the following:
- //
- // 1. Removes entries from RUNNING_CACHE and
LONG_RUNNING_COMPACTIONS_BY_RG that are not really
- // running
- // 2. Cancels running compactions for groups that are not in the current
configuration
- // 3. Remove groups not in configuration from TIME_COMPACTOR_LAST_CHECKED
- // 4. Log groups with no compactors
- // 5. Log compactors with no groups
- // 6. Log groups with compactors and queued jos that have not checked in
-
+ public void resizeThreadPools() {
var config = ctx.getConfiguration();
ThreadPools.resizePool(reservationPools.get(DataLevel.ROOT), config,
Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_ROOT);
@@ -1097,81 +949,5 @@ public class CompactionCoordinator
Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_META);
ThreadPools.resizePool(reservationPools.get(DataLevel.USER), config,
Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_USER);
-
- // grab a snapshot of the ids in the set before reading the metadata
table. This is done to
- // avoid removing things that are added while reading the metadata.
- final Set<ExternalCompactionId> idsSnapshot =
Set.copyOf(RUNNING_CACHE.keySet());
-
- // grab the ids that are listed as running in the metadata table. It
important that this is done
- // after getting the snapshot.
- final Set<ExternalCompactionId> idsInMetadata =
readExternalCompactionIds();
- LOG.trace("Current ECIDs in metadata: {}", idsInMetadata.size());
- LOG.trace("Current ECIDs in running cache: {}", idsSnapshot.size());
-
- final Set<ExternalCompactionId> idsToRemove = Sets.difference(idsSnapshot,
idsInMetadata);
-
- // remove ids that are in the running set but not in the metadata table
- idsToRemove.forEach(this::recordCompletion);
- if (idsToRemove.size() > 0) {
- LOG.debug("Removed stale entries from RUNNING_CACHE : {}", idsToRemove);
- }
-
- // Get the set of groups being referenced in the current configuration
- Set<ResourceGroupId> groupsInConfiguration = null;
- try {
- groupsInConfiguration = getCompactionServicesConfigurationGroups();
- } catch (RuntimeException | ReflectiveOperationException e) {
- LOG.error(
- "Error getting groups from the compaction services configuration.
Unable to clean up internal state.",
- e);
- return;
- }
-
- // Compaction jobs are created in the TabletGroupWatcher and added to the
Coordinator
- // via the addJobs method which adds the job to the CompactionJobQueues
object.
- final Set<ResourceGroupId> groupsWithJobs = jobQueues.getQueueIds();
-
- final Set<ResourceGroupId> jobGroupsNotInConfiguration =
- Sets.difference(groupsWithJobs, groupsInConfiguration);
-
- if (jobGroupsNotInConfiguration != null &&
!jobGroupsNotInConfiguration.isEmpty()) {
- RUNNING_CACHE.values().forEach(tec -> {
- if
(jobGroupsNotInConfiguration.contains(ResourceGroupId.of(tec.getGroupName()))) {
- LOG.warn(
- "External compaction {} running in group {} on compactor {},"
- + " but group not found in current configuration. Failing
compaction...",
- tec.getJob().getExternalCompactionId(), tec.getGroupName(),
tec.getCompactor());
- cancelCompactionOnCompactor(tec.getCompactor(),
tec.getJob().getExternalCompactionId());
- }
- });
-
- }
-
- final Set<ServerId> runningCompactors = getRunningCompactors();
-
- final Set<ResourceGroupId> runningCompactorGroups = new HashSet<>();
- runningCompactors.forEach(
- c ->
runningCompactorGroups.add(ResourceGroupId.of(c.getResourceGroup().canonical())));
-
- final Set<ResourceGroupId> groupsWithNoCompactors =
- Sets.difference(groupsInConfiguration, runningCompactorGroups);
- if (groupsWithNoCompactors != null && !groupsWithNoCompactors.isEmpty()) {
- for (ResourceGroupId group : groupsWithNoCompactors) {
- long queuedJobCount = jobQueues.getQueuedJobs(group);
- if (queuedJobCount > 0) {
- LOG.warn("Compactor group {} has {} queued compactions but no
running compactors", group,
- queuedJobCount);
- }
- }
- }
-
- final Set<ResourceGroupId> compactorsWithNoGroups =
- Sets.difference(runningCompactorGroups, groupsInConfiguration);
- if (compactorsWithNoGroups != null && !compactorsWithNoGroups.isEmpty()) {
- LOG.warn(
- "The following groups have running compactors, but are not in the
current configuration: {}",
- compactorsWithNoGroups);
- }
-
}
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java
index 3933a62504..89cdedab8e 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java
@@ -20,7 +20,6 @@ package
org.apache.accumulo.manager.compaction.coordinator.commit;
import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.fate.Repo;
-import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.manager.tableOps.AbstractFateOperation;
import org.apache.accumulo.manager.tableOps.FateEnv;
@@ -41,7 +40,6 @@ public class PutGcCandidates extends AbstractFateOperation {
env.getContext().getAmple().putGcCandidates(commitData.getTableId(),
commitData.getJobFiles());
if (refreshLocation == null) {
- env.recordCompactionCompletion(ExternalCompactionId.of(commitData.ecid));
return null;
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java
index d3be75c012..cdcf649e02 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java
@@ -27,7 +27,6 @@ import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.fate.Repo;
import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.manager.tableOps.AbstractFateOperation;
import org.apache.accumulo.manager.tableOps.FateEnv;
@@ -63,8 +62,6 @@ public class RefreshTablet extends AbstractFateOperation {
executorService.shutdownNow();
}
- env.recordCompactionCompletion(ExternalCompactionId.of(compactionId));
-
return null;
}
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java
b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java
index e7909dd0a7..01cf2baea2 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java
@@ -28,18 +28,15 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.Ample;
-import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.accumulo.core.util.time.SteadyTime;
@@ -176,26 +173,6 @@ public class FateWorkerEnv implements FateEnv {
return eventHandler;
}
- @Override
- public void recordCompactionCompletion(ExternalCompactionId ecid) {
- var coordinatorHost =
ExternalCompactionUtil.findCompactionCoordinator(getContext());
- if (coordinatorHost.isPresent()) {
- CompactionCoordinatorService.Client client = null;
- try {
- client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR,
coordinatorHost.orElseThrow(),
- getContext());
- client.recordCompletion(TraceUtil.traceInfo(),
getContext().rpcCreds(), ecid.canonical());
- log.trace("Sent compaction completion {} {}", coordinatorHost, ecid);
- } catch (TException te) {
- log.trace("Failed to send compaction completion {} {}",
coordinatorHost, ecid, te);
- } finally {
- ThriftUtil.returnClient(client, getContext());
- }
- } else {
- log.trace("No coordinator found, dropping compaction completion for {}",
ecid);
- }
- }
-
@Override
public Set<TServerInstance> onlineTabletServers() {
return liveTServerSet.getSnapshot().getTservers();
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java
index f64b3851ae..9eece1208f 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java
@@ -23,7 +23,6 @@ import java.util.concurrent.ExecutorService;
import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.util.time.SteadyTime;
import org.apache.accumulo.manager.EventPublisher;
import org.apache.accumulo.manager.split.FileRangeCache;
@@ -36,8 +35,6 @@ public interface FateEnv {
EventPublisher getEventPublisher();
- void recordCompactionCompletion(ExternalCompactionId ecid);
-
Set<TServerInstance> onlineTabletServers();
TableManager getTableManager();
diff --git
a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
index 3aa3f848d6..fcb8d9afb1 100644
---
a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.accumulo.manager.compaction;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
@@ -32,7 +31,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@@ -40,7 +38,6 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.admin.CompactionConfig;
-import org.apache.accumulo.core.client.admin.servers.ServerId;
import org.apache.accumulo.core.clientImpl.thrift.TInfo;
import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
import org.apache.accumulo.core.compaction.thrift.TCompactionState;
@@ -84,14 +81,10 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
-import com.google.common.net.HostAndPort;
-
public class CompactionCoordinatorTest {
private static final ResourceGroupId GROUP_ID = ResourceGroupId.of("R2DQ");
- private final HostAndPort tserverAddr = HostAndPort.fromParts("192.168.1.1",
9090);
-
public MetricsInfo getMockMetrics() {
MetricsInfo metricsInfo = createMock(MetricsInfo.class);
metricsInfo.addMetricsProducers(anyObject());
@@ -150,34 +143,10 @@ public class CompactionCoordinatorTest {
metadataCompactionIds = mci;
}
- @Override
- protected Set<ExternalCompactionId> readExternalCompactionIds() {
- if (metadataCompactionIds == null) {
- return RUNNING_CACHE.keySet();
- } else {
- return metadataCompactionIds;
- }
- }
-
- public Map<ExternalCompactionId,TExternalCompaction> getRunning() {
- return RUNNING_CACHE;
- }
-
public void resetInternals() {
- getRunning().clear();
metadataCompactionIds = null;
}
- @Override
- protected List<TExternalCompaction> getCompactionsRunningOnCompactors() {
- return runningCompactions;
- }
-
- @Override
- protected Set<ServerId> getRunningCompactors() {
- return Set.of();
- }
-
@Override
protected CompactionMetadata reserveCompaction(ResolvedCompactionJob rcJob,
String compactorAddress, ExternalCompactionId externalCompactionId) {
@@ -206,10 +175,6 @@ public class CompactionCoordinatorTest {
.toThrift(),
Map.of());
}
-
- @Override
- protected void cancelCompactionOnCompactor(String address, String
externalCompactionId) {}
-
}
private TableId tableId;
@@ -259,50 +224,10 @@ public class CompactionCoordinatorTest {
public void testCoordinatorColdStart() throws Exception {
var coordinator = new TestCoordinator(manager, new ArrayList<>());
assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
- assertEquals(0, coordinator.getRunning().size());
coordinator.run();
coordinator.shutdown();
assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
- assertEquals(0, coordinator.getRunning().size());
- }
-
- @Test
- public void testCoordinatorRestartOneRunningCompaction() throws Exception {
- List<TExternalCompaction> runningCompactions = new ArrayList<>();
- ExternalCompactionId eci =
ExternalCompactionId.generate(UUID.randomUUID());
-
- TExternalCompactionJob job = createMock(TExternalCompactionJob.class);
-
expect(job.getExternalCompactionId()).andReturn(eci.toString()).atLeastOnce();
- TKeyExtent extent = new TKeyExtent();
- extent.setTable("1".getBytes(UTF_8));
-
- TExternalCompaction current = new TExternalCompaction();
- current.setCompactor(tserverAddr.toString());
- current.setGroupName(GROUP_ID.canonical());
- current.setJob(job);
-
- runningCompactions.add(current);
-
- replay(job);
-
- var coordinator = new TestCoordinator(manager, runningCompactions);
- coordinator.resetInternals();
- assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
- assertEquals(0, coordinator.getRunning().size());
- coordinator.run();
- coordinator.shutdown();
- assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
- assertEquals(1, coordinator.getRunning().size());
-
- Map<ExternalCompactionId,TExternalCompaction> running =
coordinator.getRunning();
- Entry<ExternalCompactionId,TExternalCompaction> ecomp =
running.entrySet().iterator().next();
- assertEquals(eci, ecomp.getKey());
- TExternalCompaction tec = ecomp.getValue();
- assertEquals(GROUP_ID, ResourceGroupId.of(tec.getGroupName()));
- assertEquals(tserverAddr.toString(), tec.getCompactor());
-
- verify(job);
}
@Test
@@ -318,14 +243,12 @@ public class CompactionCoordinatorTest {
var coordinator = new TestCoordinator(manager, new ArrayList<>());
assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
- assertEquals(0, coordinator.getRunning().size());
// Use coordinator.run() to populate the internal data structures. This is
tested in a different
// test.
coordinator.run();
coordinator.shutdown();
assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
- assertEquals(0, coordinator.getRunning().size());
// Add a job to the job queue
CompactionJob job =
@@ -344,12 +267,6 @@ public class CompactionCoordinatorTest {
assertEquals(ke, KeyExtent.fromThrift(createdJob.getExtent()));
assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
- assertEquals(1, coordinator.getRunning().size());
- Entry<ExternalCompactionId,TExternalCompaction> entry =
- coordinator.getRunning().entrySet().iterator().next();
- assertEquals(eci.toString(), entry.getKey().toString());
- assertEquals("localhost:10241", entry.getValue().getCompactor());
- assertEquals(eci.toString(),
entry.getValue().getJob().getExternalCompactionId());
verify(tm);
}
@@ -362,32 +279,4 @@ public class CompactionCoordinatorTest {
assertEquals(3, nextJob.getCompactorCount());
assertNull(nextJob.getJob().getExternalCompactionId());
}
-
- @Test
- public void testCleanUpRunning() throws Exception {
- TExternalCompaction ext1 = createMock(TExternalCompaction.class);
- TExternalCompaction ext2 = createMock(TExternalCompaction.class);
- TExternalCompaction ext3 = createMock(TExternalCompaction.class);
- replay(ext1, ext2, ext3);
-
- TestCoordinator coordinator = new TestCoordinator(manager, new
ArrayList<>());
-
- var ecid1 = ExternalCompactionId.generate(UUID.randomUUID());
- var ecid2 = ExternalCompactionId.generate(UUID.randomUUID());
- var ecid3 = ExternalCompactionId.generate(UUID.randomUUID());
-
- coordinator.getRunning().put(ecid1, ext1);
- coordinator.getRunning().put(ecid2, ext2);
- coordinator.getRunning().put(ecid3, ext3);
- coordinator.cleanUpInternalState();
-
- assertEquals(Set.of(ecid1, ecid2, ecid3),
coordinator.getRunning().keySet());
-
- coordinator.setMetadataCompactionIds(Set.of(ecid1, ecid2));
- coordinator.cleanUpInternalState();
-
- assertEquals(Set.of(ecid1, ecid2), coordinator.getRunning().keySet());
-
- verify(ext1, ext2, ext3);
- }
}
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java
index cb67c61fc4..9b27141f0d 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java
@@ -53,6 +53,7 @@ import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.compaction.CompactionPluginUtils;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.eclipse.jetty.util.NanoTime;
import org.slf4j.Logger;
@@ -305,6 +306,15 @@ public class InformationFetcher implements
RemovalListener<ServerId,MetricRespon
futures.add(this.pool.submit(new TableInformationFetcher(this.ctx,
tableId, summary)));
}
+ futures.add(this.pool.submit(() -> {
+ try {
+ var groups =
CompactionPluginUtils.getConfiguredCompactionResourceGroups(ctx);
+ summary.addConfiguredCompactionGroups(groups);
+ } catch (ReflectiveOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }));
+
long monitorFetchTimeout =
ctx.getConfiguration().getTimeInMillis(Property.MONITOR_FETCH_TIMEOUT);
long allFuturesAdded = NanoTime.now();
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java
index 2a5148010f..3de740a1b8 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java
@@ -415,6 +415,8 @@ public class SystemInformation {
private final Set<String> suggestions = new ConcurrentSkipListSet<>();
+ private final Set<String> configuredCompactionResourceGroups =
ConcurrentHashMap.newKeySet();
+
private long timestamp = 0;
private ScanServerView scanServerView;
private final int rgLongRunningCompactionSize;
@@ -448,6 +450,7 @@ public class SystemInformation {
suggestions.clear();
runningCompactionsPerGroup.clear();
runningCompactionsPerTable.clear();
+ configuredCompactionResourceGroups.clear();
scanServerView = null;
}
@@ -580,6 +583,10 @@ public class SystemInformation {
problemHosts.add(server);
}
+ public void addConfiguredCompactionGroups(Set<String> groups) {
+ configuredCompactionResourceGroups.addAll(groups);
+ }
+
public void finish() {
// Update the deployment not-responded numbers based
// on the problem hosts.
@@ -627,6 +634,14 @@ public class SystemInformation {
}
}
}
+
+ for (var compactorGroup : compactors.keySet()) {
+ if (!configuredCompactionResourceGroups.contains(compactorGroup)) {
+ suggestions.add("Compactor group " + compactorGroup
+ + " has running compactors, but no configuration uses them.");
+ }
+ }
+
Set<ServerId> scanServers = new HashSet<>();
sservers.values().forEach(scanServers::addAll);
int problemScanServerCount = (int) problemHosts.stream()
