This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 304f52c0f38287095e72375c4324a4f5031c9b2b Merge: aab373f47c 9961765565 Author: Daniel Roberts ddanielr <[email protected]> AuthorDate: Mon Apr 13 19:25:16 2026 +0000 Merge branch '2.1' .../org/apache/accumulo/core/conf/Property.java | 8 + .../accumulo/server/util/MetadataTableUtil.java | 2 +- .../accumulo/manager/tableOps/delete/CleanUp.java | 32 +- .../functional/DeleteAndVerifyFileRemovalsIT.java | 390 +++++++++++++++++++++ 4 files changed, 417 insertions(+), 15 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java index 1bbd0f0817,3d04375d84..cdf70a19a3 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@@ -420,29 -391,29 +420,37 @@@ public enum Property "Maximum number of threads the TabletGroupWatcher will use in its BatchScanner to" + " look for tablets that need maintenance.", "2.1.4"), + MANAGER_TABLET_REFRESH_MINTHREADS("manager.tablet.refresh.threads.minimum", "10", + PropertyType.COUNT, + """ + The Manager will notify TabletServers that a Tablet needs to be refreshed after certain operations \ + are performed (e.g. Bulk Import). This property specifies the number of core threads in a \ + ThreadPool in the Manager that will be used to request these refresh operations. + """, + "4.0.0"), + MANAGER_TABLET_REFRESH_MAXTHREADS("manager.tablet.refresh.threads.maximum", "10", + PropertyType.COUNT, + """ + The Manager will notify TabletServers that a Tablet needs to be refreshed after certain operations \ + are performed (e.g. Bulk Import). This property specifies the maximum number of threads in a \ + ThreadPool in the Manager that will be used to request these refresh operations. + """, + "4.0.0"), + MANAGER_TABLET_MERGEABILITY_INTERVAL("manager.tablet.mergeability.interval", "24h", + PropertyType.TIMEDURATION, + "Time to wait between scanning tables to identify ranges of tablets that can be " + + " auto-merged. Valid ranges will be have merge fate ops submitted.", + "4.0.0"), + MANAGER_TABLE_DELETE_OPTIMIZATION("manager.table.delete.optimization", "true", + PropertyType.BOOLEAN, + "When deleting a table the Manager will remove related table directories from " + + " the storage volumes if there are no other references to the files in the " + + " metadata table. When deleting a lot of tables this optimization can be costly. " + + " Setting this value to false will skip this optimization and the table directory " + + " cleanup will occur in the Garbage Collector instead.", + "2.1.5"), - MANAGER_BULK_RETRIES("manager.bulk.retries", "3", PropertyType.COUNT, - "The number of attempts to bulk import a RFile before giving up.", "1.4.0"), - MANAGER_BULK_THREADPOOL_SIZE("manager.bulk.threadpool.size", "5", PropertyType.COUNT, - "The number of threads to use when coordinating a bulk import.", "1.4.0"), - MANAGER_BULK_THREADPOOL_TIMEOUT("manager.bulk.threadpool.timeout", "0s", - PropertyType.TIMEDURATION, - "The time after which bulk import threads terminate with no work available. Zero (0) will keep the threads alive indefinitely.", - "2.1.0"), MANAGER_BULK_TIMEOUT("manager.bulk.timeout", "5m", PropertyType.TIMEDURATION, "The time to wait for a tablet server to process a bulk import request.", "1.4.3"), - MANAGER_BULK_MAX_CONNECTIONS("manager.bulk.connections.max", "8", PropertyType.COUNT, - "The maximum number of connections the manager can make to a single tablet server for bulkv2 " - + "load request. For the case where a single tablet server has a lot of tablets for a bulk import " - + "increasing this may help lower the time it takes to load those tablets.", - "2.1.4"), MANAGER_RENAME_THREADS("manager.rename.threadpool.size", "20", PropertyType.COUNT, "The number of threads to use when renaming user files during table import or bulk ingest.", "2.1.0"), diff --cc server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java index 3f8d6baabe,7362fd84e4..a2b4ddb40a --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java @@@ -23,10 -23,11 +23,10 @@@ import java.net.UnknownHostException import java.util.Arrays; import java.util.Map.Entry; - import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; + import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.Range; @@@ -64,14 -83,55 +64,18 @@@ class CleanUp extends AbstractFateOpera } @Override - public Repo<Manager> call(long tid, Manager manager) { - - manager.clearMigrations(tableId); - + public Repo<FateEnv> call(FateId fateId, FateEnv env) { int refCount = 0; - try { - if (!manager.getConfiguration().getBoolean(Property.MANAGER_TABLE_DELETE_OPTIMIZATION)) { ++ if (!env.getContext().getConfiguration() ++ .getBoolean(Property.MANAGER_TABLE_DELETE_OPTIMIZATION)) { + // Skip scanning the metadata table for each table delete and always allow the GC to handle + // file deletion. + refCount = -1; + } else { // look for other tables that references this table's files - AccumuloClient client = env.getContext(); -- try (BatchScanner bs = - client.createBatchScanner(SystemTables.METADATA.tableName(), Authorizations.EMPTY, 8)) { - manager.getContext().createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY, 8)) { ++ try (BatchScanner bs = env.getContext().createBatchScanner(SystemTables.METADATA.tableName(), ++ Authorizations.EMPTY, 8)) { Range allTables = TabletsSection.getRange(); Range tableRange = TabletsSection.getRange(tableId); Range beforeTable = @@@ -85,15 -145,15 +89,15 @@@ for (Entry<Key,Value> entry : bs) { if (entry.getKey().getColumnQualifier().toString().contains("/" + tableId + "/")) { - refCount++; + refCount = 1; + break; } } + } catch (Exception e) { + refCount = -1; + log.error("Failed to scan {} looking for references to deleted table {}", - MetadataTable.NAME, tableId, e); ++ SystemTables.METADATA.tableName(), tableId, e); } - - } catch (Exception e) { - refCount = -1; - log.error("Failed to scan " + SystemTables.METADATA.tableName() - + " looking for references to deleted table " + tableId, e); } // remove metadata table entries @@@ -103,16 -163,23 +107,16 @@@ // If the manager lock passed to deleteTable, it is possible that the delete mutations will be // dropped. If the delete operations // are dropped and the operation completes, then the deletes will not be repeated. - MetadataTableUtil.deleteTable(tableId, refCount != 0, manager.getContext(), null); + MetadataTableUtil.deleteTable(tableId, refCount != 0, env.getContext(), null); } catch (Exception e) { - log.error("error deleting " + tableId + " from metadata table", e); + log.error("error deleting {} from metadata table", tableId, e); } - // remove any problem reports the table may have - try { - ProblemReports.getInstance(manager.getContext()).deleteProblemReports(tableId); - } catch (Exception e) { - log.error("Failed to delete problem reports for table {}", tableId, e); - } - if (refCount == 0) { - // delete the map files + // delete the data files try { - VolumeManager fs = manager.getVolumeManager(); - for (String dir : manager.getContext().getTablesDirs()) { + VolumeManager fs = env.getVolumeManager(); + for (String dir : env.getContext().getTablesDirs()) { fs.deleteRecursively(new Path(dir, tableId.canonical())); } } catch (IOException e) { @@@ -129,10 -196,10 +133,10 @@@ // remove table from zookeeper try { - manager.getTableManager().removeTable(tableId); - manager.getContext().clearTableListCache(); + env.getTableManager().removeTable(tableId, namespaceId); + env.getContext().clearTableListCache(); } catch (Exception e) { - log.error("Failed to find table id in zookeeper", e); + log.error("Failed to find table id {} in zookeeper", tableId, e); } // remove any permissions associated with this table @@@ -143,10 -210,10 +147,10 @@@ log.error("{}", e.getMessage(), e); } - Utils.unreserveTable(manager, tableId, tid, true); - Utils.unreserveNamespace(manager, namespaceId, tid, false); + Utils.unreserveTable(env.getContext(), tableId, fateId, LockType.WRITE); + Utils.unreserveNamespace(env.getContext(), namespaceId, fateId, LockType.READ); - LoggerFactory.getLogger(CleanUp.class).debug("Deleted table " + tableId); + log.debug("Deleted table {}", tableId); return null; }
