This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9c7c467a5614c095d4a78c3bd0e8440841e5669b
Merge: 9fa27a6d4d c4c8c572c8
Author: Daniel Roberts <ddani...@gmail.com>
AuthorDate: Wed Oct 25 03:45:44 2023 +0000
Merge branch 'main' into elasticity
.../org/apache/accumulo/core/conf/Property.java | 246 ++++++++++-----------
.../org/apache/accumulo/core/gc/Reference.java | 7 +
.../accumulo/core/gc/ReferenceDirectory.java | 2 +-
.../org/apache/accumulo/core/gc/ReferenceFile.java | 32 ++-
.../apache/accumulo/core/conf/PropertyTest.java | 4 +
.../accumulo/server/gc/AllVolumesDirectory.java | 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 4 +-
.../accumulo/server/util/MetadataTableUtil.java | 2 +-
.../main/java/org/apache/accumulo/gc/GCRun.java | 31 +--
.../accumulo/gc/GarbageCollectionAlgorithm.java | 16 +-
.../apache/accumulo/gc/GarbageCollectionTest.java | 164 +++++++++-----
.../tableOps/bulkVer2/CleanUpBulkImport.java | 2 +-
.../test/functional/GarbageCollectorIT.java | 2 +-
13 files changed, 293 insertions(+), 221 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 0b4df706b0,58be89df05..d2dea79464
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -298,19 -297,16 +298,19 @@@ public enum Property
"3.0.0"),
GENERAL_MAX_SCANNER_RETRY_PERIOD("general.max.scanner.retry.period", "5s",
PropertyType.TIMEDURATION,
- "The maximum amount of time that a Scanner should wait before retrying
a failed RPC",
+ "The maximum amount of time that a Scanner should wait before retrying
a failed RPC.",
"1.7.3"),
+
GENERAL_MICROMETER_CACHE_METRICS_ENABLED("general.micrometer.cache.metrics.enabled",
"false",
- PropertyType.BOOLEAN, "Enables Caffeine Cache metrics functionality
using Micrometer",
++ PropertyType.BOOLEAN, "Enables Caffeine Cache metrics functionality
using Micrometer.",
+ "4.0.0"),
GENERAL_MICROMETER_ENABLED("general.micrometer.enabled", "false",
PropertyType.BOOLEAN,
- "Enables metrics functionality using Micrometer", "2.1.0"),
+ "Enables metrics functionality using Micrometer.", "2.1.0"),
GENERAL_MICROMETER_JVM_METRICS_ENABLED("general.micrometer.jvm.metrics.enabled",
"false",
- PropertyType.BOOLEAN, "Enables JVM metrics functionality using
Micrometer", "2.1.0"),
+ PropertyType.BOOLEAN, "Enables JVM metrics functionality using
Micrometer.", "2.1.0"),
GENERAL_MICROMETER_FACTORY("general.micrometer.factory", "",
PropertyType.CLASSNAME,
- "Name of class that implements MeterRegistryFactory", "2.1.0"),
+ "Name of class that implements MeterRegistryFactory.", "2.1.0"),
GENERAL_PROCESS_BIND_ADDRESS("general.process.bind.addr", "0.0.0.0",
PropertyType.STRING,
- "The local IP address to which this server should bind for sending and
receiving network traffic",
+ "The local IP address to which this server should bind for sending and
receiving network traffic.",
"3.0.0"),
// properties that are specific to manager server behavior
MANAGER_PREFIX("manager.", null, PropertyType.PREFIX,
@@@ -326,13 -322,8 +326,13 @@@
PropertyType.TIMEDURATION,
"Time to wait between scanning tablet states to identify tablets that
need to be assigned, un-assigned, migrated, etc.",
"2.1.2"),
+
MANAGER_TABLET_GROUP_WATCHER_SCAN_THREADS("manager.tablet.watcher.scan.threads.max",
"16",
+ PropertyType.COUNT,
+ "Maximum number of threads the TabletGroupWatcher will use in its
BatchScanner to"
- + " look for tablets that need maintenance",
++ + " look for tablets that need maintenance.",
+ "4.0.0"),
MANAGER_BULK_TIMEOUT("manager.bulk.timeout", "5m",
PropertyType.TIMEDURATION,
- "The time to wait for a tablet server to process a bulk import
request", "1.4.3"),
+ "The time to wait for a tablet server to process a bulk import
request.", "1.4.3"),
MANAGER_RENAME_THREADS("manager.rename.threadpool.size", "20",
PropertyType.COUNT,
"The number of threads to use when renaming user files during table
import or bulk ingest.",
"2.1.0"),
@@@ -383,13 -374,8 +383,13 @@@
"Maximum time manager will wait for tserver available threshold "
+ "to be reached before continuing. When set to 0 or less, will
block "
+ "indefinitely. Default is 0 to block indefinitely. Only valid
when tserver available "
- + "threshold is set greater than 0. Added with version 1.10",
+ + "threshold is set greater than 0.",
"1.10.0"),
+ MANAGER_SPLIT_WORKER_THREADS("manager.split.inspection.threadpool.size",
"8", PropertyType.COUNT,
+ "The number of threads used to inspect tablets files to find split
points.", "4.0.0"),
+
+
MANAGER_COMPACTION_SERVICE_PRIORITY_QUEUE_SIZE("manager.compaction.major.service.queue.size",
- "10000", PropertyType.COUNT, "The max size of the priority queue",
"4.0"),
++ "10000", PropertyType.COUNT, "The max size of the priority queue.",
"4.0"),
// properties that are specific to scan server behavior
@Experimental
SSERV_PREFIX("sserver.", null, PropertyType.PREFIX,
@@@ -580,53 -565,72 +580,53 @@@
"Max time for the thrift client handler to wait for scan results before
timing out.",
"2.1.0"),
TSERV_MIGRATE_MAXCONCURRENT("tserver.migrations.concurrent.max", "1",
PropertyType.COUNT,
- "The maximum number of concurrent tablet migrations for a tablet
server", "1.3.5"),
+ "The maximum number of concurrent tablet migrations for a tablet
server.", "1.3.5"),
+ // ELASTICITY_TODO look into removing this prop, may need to deprecate in
3.0
TSERV_MAJC_DELAY("tserver.compaction.major.delay", "30s",
PropertyType.TIMEDURATION,
"Time a tablet server will sleep between checking which tablets need
compaction.", "1.3.5"),
TSERV_COMPACTION_SERVICE_PREFIX("tserver.compaction.major.service.", null,
PropertyType.PREFIX,
"Prefix for compaction services.", "2.1.0"),
TSERV_COMPACTION_SERVICE_ROOT_PLANNER("tserver.compaction.major.service.root.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
- "Compaction planner for root tablet service", "2.1.0"),
+ "Compaction planner for root tablet service.", "2.1.0"),
- @Deprecated(since = "3.1", forRemoval = true)
-
TSERV_COMPACTION_SERVICE_ROOT_RATE_LIMIT("tserver.compaction.major.service.root.rate.limit",
"0B",
- PropertyType.BYTES,
- "Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
- "2.1.0"),
TSERV_COMPACTION_SERVICE_ROOT_MAX_OPEN(
"tserver.compaction.major.service.root.planner.opts.maxOpen", "30",
PropertyType.COUNT,
- "The maximum number of files a compaction will open", "2.1.0"),
+ "The maximum number of files a compaction will open.", "2.1.0"),
TSERV_COMPACTION_SERVICE_ROOT_EXECUTORS(
"tserver.compaction.major.service.root.planner.opts.executors",
-
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':1},{'name':'huge','type':'internal','numThreads':1}]"
- .replaceAll("'", "\""),
+
"[{'name':'all','type':'external','group':'accumulo_meta'}]".replaceAll("'",
"\""),
PropertyType.STRING,
- "See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %} ",
+ "See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"2.1.0"),
TSERV_COMPACTION_SERVICE_META_PLANNER("tserver.compaction.major.service.meta.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
- "Compaction planner for metadata table", "2.1.0"),
+ "Compaction planner for metadata table.", "2.1.0"),
- @Deprecated(since = "3.1", forRemoval = true)
-
TSERV_COMPACTION_SERVICE_META_RATE_LIMIT("tserver.compaction.major.service.meta.rate.limit",
"0B",
- PropertyType.BYTES,
- "Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
- "2.1.0"),
TSERV_COMPACTION_SERVICE_META_MAX_OPEN(
"tserver.compaction.major.service.meta.planner.opts.maxOpen", "30",
PropertyType.COUNT,
- "The maximum number of files a compaction will open", "2.1.0"),
+ "The maximum number of files a compaction will open.", "2.1.0"),
TSERV_COMPACTION_SERVICE_META_EXECUTORS(
"tserver.compaction.major.service.meta.planner.opts.executors",
-
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':2},{'name':'huge','type':'internal','numThreads':2}]"
- .replaceAll("'", "\""),
+
"[{'name':'all','type':'external','group':'accumulo_meta'}]".replaceAll("'",
"\""),
PropertyType.STRING,
- "See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %} ",
+ "See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"2.1.0"),
TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER("tserver.compaction.major.service.default.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Planner for default compaction service.", "2.1.0"),
- @Deprecated(since = "3.1", forRemoval = true)
-
TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT("tserver.compaction.major.service.default.rate.limit",
- "0B", PropertyType.BYTES,
- "Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
- "2.1.0"),
TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN(
"tserver.compaction.major.service.default.planner.opts.maxOpen", "10",
PropertyType.COUNT,
- "The maximum number of files a compaction will open", "2.1.0"),
+ "The maximum number of files a compaction will open.", "2.1.0"),
TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS(
"tserver.compaction.major.service.default.planner.opts.executors",
-
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':2},{'name':'medium','type':'internal','maxSize':'128M','numThreads':2},{'name':'large','type':'internal','numThreads':2}]"
+
("[{'name':'small','type':'external','maxSize':'128M','group':'user_small'},
{'name':'large','type':'external','group':'user_large'}]")
.replaceAll("'", "\""),
PropertyType.STRING,
- "See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %} ",
+ "See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"2.1.0"),
TSERV_MINC_MAXCONCURRENT("tserver.compaction.minor.concurrent.max", "4",
PropertyType.COUNT,
- "The maximum number of concurrent minor compactions for a tablet
server", "1.3.5"),
+ "The maximum number of concurrent minor compactions for a tablet
server.", "1.3.5"),
TSERV_COMPACTION_WARN_TIME("tserver.compaction.warn.time", "10m",
PropertyType.TIMEDURATION,
- "When a compaction has not made progress for this time period, a
warning will be logged",
+ "When a compaction has not made progress for this time period, a
warning will be logged.",
"1.6.0"),
TSERV_BLOOM_LOAD_MAXCONCURRENT("tserver.bloom.load.concurrent.max", "4",
PropertyType.COUNT,
"The number of concurrent threads that will load bloom filters in the
background. "
@@@ -731,14 -736,6 +731,14 @@@
+ " tserver is available before tablets are initially assigned if
the 'last' location is"
+ " to be used.",
"2.1.1"),
+
TSERV_ONDEMAND_UNLOADER_INTERVAL("tserver.ondemand.tablet.unloader.interval",
"10m",
+ PropertyType.TIMEDURATION,
- "The interval at which the TabletServer will check if on-demand tablets
can be unloaded",
++ "The interval at which the TabletServer will check if on-demand tablets
can be unloaded.",
+ "4.0.0"),
+ TSERV_GROUP_NAME("tserver.group", Constants.DEFAULT_RESOURCE_GROUP_NAME,
PropertyType.STRING,
+ "Resource group name for this TabletServer. Resource groups can be
defined to dedicate resources "
- + " to specific tables (e.g. balancing tablets for table(s) within
a group, see TableLoadBalancer)",
++ + " to specific tables (e.g. balancing tablets for table(s) within
a group, see TableLoadBalancer).",
+ "4.0.0"),
// accumulo garbage collector properties
GC_PREFIX("gc.", null, PropertyType.PREFIX,
@@@ -877,11 -874,7 +877,11 @@@
"A plugin that can dynamically configure compaction output files based
on input files.",
"2.1.0"),
TABLE_COMPACTION_CONFIGURER_OPTS("table.compaction.configurer.opts.", null,
PropertyType.PREFIX,
- "Options for the table compaction configuror", "2.1.0"),
+ "Options for the table compaction configuror.", "2.1.0"),
+ TABLE_ONDEMAND_UNLOADER("tserver.ondemand.tablet.unloader",
+ "org.apache.accumulo.core.spi.ondemand.DefaultOnDemandTabletUnloader",
PropertyType.CLASSNAME,
+ "The class that will be used to determine which on-demand Tablets to
unload.", "4.0.0"),
+
// Crypto-related properties
@Experimental
TABLE_CRYPTO_PREFIX("table.crypto.opts.", null, PropertyType.PREFIX,
@@@ -1094,20 -1088,12 +1094,20 @@@
COMPACTOR_PREFIX("compactor.", null, PropertyType.PREFIX,
"Properties in this category affect the behavior of the accumulo
compactor server.", "2.1.0"),
@Experimental
- COMPACTOR_PORTSEARCH("compactor.port.search", "false", PropertyType.BOOLEAN,
+ COMPACTOR_PORTSEARCH("compactor.port.search", "true", PropertyType.BOOLEAN,
- "If the compactor.port.client is in use, search higher ports until one
is available",
+ "If the compactor.port.client is in use, search higher ports until one
is available.",
"2.1.0"),
@Experimental
COMPACTOR_CLIENTPORT("compactor.port.client", "9133", PropertyType.PORT,
- "The port used for handling client connections on the compactor
servers", "2.1.0"),
+ "The port used for handling client connections on the compactor
servers.", "2.1.0"),
+ COMPACTOR_MIN_JOB_WAIT_TIME("compactor.wait.time.job.min", "1s",
PropertyType.TIMEDURATION,
+ "The minimum amount of time to wait between checks for the next
compaction job, backing off"
+ + "exponentially until COMPACTOR_MAX_JOB_WAIT_TIME is reached.",
+ "4.0.0"),
+ COMPACTOR_MAX_JOB_WAIT_TIME("compactor.wait.time.job.max", "5m",
PropertyType.TIMEDURATION,
+ "Compactors do exponential backoff when their request for work
repeatedly come back empty. "
+ + "This is the maximum amount of time to wait between checks for
the next compaction job.",
+ "4.0.0"),
@Experimental
COMPACTOR_MINTHREADS("compactor.threads.minimum", "1", PropertyType.COUNT,
"The minimum number of threads to use to handle incoming requests.",
"2.1.0"),
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java
index 7ccfbcdf95,562423750b..0d28058822
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java
@@@ -66,15 -59,17 +66,15 @@@ public class CleanUpBulkImport extends
ample.removeBulkLoadInProgressFlag(
"/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
ample.putGcFileAndDirCandidates(info.tableId,
- Collections.singleton(new ReferenceFile(info.tableId, bulkDir)));
+ Collections.singleton(ReferenceFile.forFile(info.tableId, bulkDir)));
- if (info.tableState == TableState.ONLINE) {
- Text firstSplit = info.firstSplit == null ? null : new
Text(info.firstSplit);
- Text lastSplit = info.lastSplit == null ? null : new
Text(info.lastSplit);
+ Text firstSplit = info.firstSplit == null ? null : new
Text(info.firstSplit);
+ Text lastSplit = info.lastSplit == null ? null : new Text(info.lastSplit);
- log.debug("{} removing the metadata table markers for loaded files in
range {} {}",
- FateTxId.formatTid(tid), firstSplit, lastSplit);
+ log.debug("{} removing the metadata table markers for loaded files in
range {} {}",
+ FateTxId.formatTid(tid), firstSplit, lastSplit);
+ removeBulkLoadEntries(ample, info.tableId, tid, firstSplit, lastSplit);
- ample.removeBulkLoadEntries(info.tableId, tid, firstSplit, lastSplit);
- }
Utils.unreserveHdfsDirectory(manager, info.sourceDir, tid);
Utils.getReadLock(manager, info.tableId, tid).unlock();
// delete json renames and mapping files
