This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 94140413234b3865c063c0c5bb1e3001e84beb8e
Merge: b63afacfa6 bb6143cbe9
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Tue Jun 18 13:26:38 2024 -0400
Merge branch '2.1'
.../util/compaction/ExternalCompactionUtil.java | 9 +
.../thrift/CompactionCoordinatorService.java | 44 +-
.../core/compaction/thrift/TNextCompactionJob.java | 511 +++++++++++++++++++++
core/src/main/thrift/compaction-coordinator.thrift | 9 +-
.../coordinator/CompactionCoordinator.java | 10 +-
.../org/apache/accumulo/compactor/Compactor.java | 13 +-
6 files changed, 565 insertions(+), 31 deletions(-)
diff --cc
core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java
index f3756c90a4,ac8e15f6e5..7b8af0438b
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java
+++
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java
@@@ -29,37 -29,37 +29,37 @@@ public class CompactionCoordinatorServi
public interface Iface {
- public void
compactionCompleted(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent,
org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws
org.apache.thrift.TException;
+ public void
compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent,
org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws
org.apache.thrift.TException;
- public
org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob
getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId) throws org.apache.thrift.TException;
- public TNextCompactionJob
getCompactionJob(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId) throws org.apache.thrift.TException;
++ public TNextCompactionJob
getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId) throws org.apache.thrift.TException;
- public void
updateCompactionStatus(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId, TCompactionStatusUpdate status, long
timestamp) throws org.apache.thrift.TException;
+ public void
updateCompactionStatus(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId, TCompactionStatusUpdate status, long
timestamp) throws org.apache.thrift.TException;
- public void compactionFailed(org.apache.accumulo.core.trace.thrift.TInfo
tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) throws
org.apache.thrift.TException;
+ public void
compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) throws
org.apache.thrift.TException;
- public TExternalCompactionList
getRunningCompactions(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws
org.apache.thrift.TException;
+ public TExternalCompactionList
getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws
org.apache.thrift.TException;
- public TExternalCompactionList
getCompletedCompactions(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws
org.apache.thrift.TException;
+ public TExternalCompactionList
getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws
org.apache.thrift.TException;
- public void cancel(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId) throws org.apache.thrift.TException;
+ public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo
tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId) throws org.apache.thrift.TException;
}
public interface AsyncIface {
- public void
compactionCompleted(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent,
org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException;
+ public void
compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent,
org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException;
- public void
getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob>
resultHandler) throws org.apache.thrift.TException;
- public void getCompactionJob(org.apache.accumulo.core.trace.thrift.TInfo
tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<TNextCompactionJob> resultHandler)
throws org.apache.thrift.TException;
++ public void
getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<TNextCompactionJob> resultHandler)
throws org.apache.thrift.TException;
- public void
updateCompactionStatus(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId, TCompactionStatusUpdate status, long
timestamp, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler)
throws org.apache.thrift.TException;
+ public void
updateCompactionStatus(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId, TCompactionStatusUpdate status, long
timestamp, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler)
throws org.apache.thrift.TException;
- public void compactionFailed(org.apache.accumulo.core.trace.thrift.TInfo
tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException;
+ public void
compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException;
- public void
getRunningCompactions(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
org.apache.thrift.async.AsyncMethodCallback<TExternalCompactionList>
resultHandler) throws org.apache.thrift.TException;
+ public void
getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
org.apache.thrift.async.AsyncMethodCallback<TExternalCompactionList>
resultHandler) throws org.apache.thrift.TException;
- public void
getCompletedCompactions(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
org.apache.thrift.async.AsyncMethodCallback<TExternalCompactionList>
resultHandler) throws org.apache.thrift.TException;
+ public void
getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
org.apache.thrift.async.AsyncMethodCallback<TExternalCompactionList>
resultHandler) throws org.apache.thrift.TException;
- public void cancel(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException;
+ public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo
tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws
org.apache.thrift.TException;
}
@@@ -111,7 -111,7 +111,7 @@@
}
@Override
- public
org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob
getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId) throws org.apache.thrift.TException
- public TNextCompactionJob
getCompactionJob(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId) throws org.apache.thrift.TException
++ public TNextCompactionJob
getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId) throws org.apache.thrift.TException
{
send_getCompactionJob(tinfo, credentials, queueName, compactor,
externalCompactionId);
return recv_getCompactionJob();
@@@ -328,20 -328,20 +328,20 @@@
}
@Override
- public void
getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob>
resultHandler) throws org.apache.thrift.TException {
- public void getCompactionJob(org.apache.accumulo.core.trace.thrift.TInfo
tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<TNextCompactionJob> resultHandler)
throws org.apache.thrift.TException {
++ public void
getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<TNextCompactionJob> resultHandler)
throws org.apache.thrift.TException {
checkReady();
getCompactionJob_call method_call = new getCompactionJob_call(tinfo,
credentials, queueName, compactor, externalCompactionId, resultHandler, this,
___protocolFactory, ___transport);
this.___currentMethod = method_call;
___manager.call(method_call);
}
- public static class getCompactionJob_call extends
org.apache.thrift.async.TAsyncMethodCall<org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob>
{
+ public static class getCompactionJob_call extends
org.apache.thrift.async.TAsyncMethodCall<TNextCompactionJob> {
- private org.apache.accumulo.core.trace.thrift.TInfo tinfo;
+ private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo;
private org.apache.accumulo.core.securityImpl.thrift.TCredentials
credentials;
private java.lang.String queueName;
private java.lang.String compactor;
private java.lang.String externalCompactionId;
- public
getCompactionJob_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob>
resultHandler, org.apache.thrift.async.TAsyncClient client,
org.apache.thrift.protocol.TProtocolFactory protocolFactory, [...]
- public
getCompactionJob_call(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<TNextCompactionJob> resultHandler,
org.apache.thrift.async.TAsyncClient client,
org.apache.thrift.protocol.TProtocolFactory protocolFactory,
org.apache.thrift.transport.TNonblockingTransport tran [...]
++ public
getCompactionJob_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
java.lang.String queueName, java.lang.String compactor, java.lang.String
externalCompactionId,
org.apache.thrift.async.AsyncMethodCallback<TNextCompactionJob> resultHandler,
org.apache.thrift.async.TAsyncClient client,
org.apache.thrift.protocol.TProtocolFactory protocolFactory,
org.apache.thrift.transport.TNonblockingTransport [...]
super(client, protocolFactory, transport, resultHandler, false);
this.tinfo = tinfo;
this.credentials = credentials;
diff --cc core/src/main/thrift/compaction-coordinator.thrift
index 9a8c72d7b4,cbf3ac0d1f..e5ff211393
--- a/core/src/main/thrift/compaction-coordinator.thrift
+++ b/core/src/main/thrift/compaction-coordinator.thrift
@@@ -77,8 -85,8 +84,8 @@@ service CompactionCoordinatorService
/*
* Called by Compactor to get the next compaction job
*/
- tabletserver.TExternalCompactionJob getCompactionJob(
+ TNextCompactionJob getCompactionJob(
- 1:trace.TInfo tinfo
+ 1:client.TInfo tinfo
2:security.TCredentials credentials
3:string queueName
4:string compactor
diff --cc
server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 708e67405e,af154442ef..b5b63da26e
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@@ -96,9 -95,8 +97,10 @@@ import org.slf4j.LoggerFactory
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
+ import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.collect.Sets;
+import com.google.common.net.HostAndPort;
+import com.google.common.util.concurrent.Uninterruptibles;
public class CompactionCoordinator extends AbstractServer
implements CompactionCoordinatorService.Iface, LiveTServerSet.Listener {
@@@ -136,10 -135,11 +138,12 @@@
// Exposed for tests
protected volatile Boolean shutdown = false;
- private ScheduledThreadPoolExecutor schedExecutor;
+ private final ScheduledThreadPoolExecutor schedExecutor;
+ private final ExecutorService summariesExecutor;
+ private final LoadingCache<String,Integer> compactorCounts;
+
- protected CompactionCoordinator(ServerOpts opts, String[] args) {
+ protected CompactionCoordinator(ConfigOpts opts, String[] args) {
this(opts, args, null);
}
