This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 1f218de444eb1aff5654c55fc84659d0491ae8b8
Merge: f674e9a0b0 f26c702a87
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Tue Jan 16 12:51:49 2024 -0500
Merge remote-tracking branch 'upstream/main' into elasticity
.../core/clientImpl/TableOperationsImpl.java | 71 +++++++++---------
.../clientImpl/bulk/ConcurrentKeyExtentCache.java | 17 ++---
.../fate/zookeeper/DistributedReadWriteLock.java | 33 +++++----
.../org/apache/accumulo/core/summary/Gatherer.java | 22 +++---
.../java/org/apache/accumulo/core/util/Merge.java | 62 ++++++++--------
.../org/apache/accumulo/core/util/MergeTest.java | 31 +++++++-
.../manager/balancer/BalancerEnvironmentImpl.java | 12 ++--
.../accumulo/server/metadata/ServerAmpleImpl.java | 2 +-
.../main/java/org/apache/accumulo/gc/GCRun.java | 4 +-
.../accumulo/gc/GarbageCollectionAlgorithm.java | 84 +++++++++++-----------
.../apache/accumulo/manager/tableOps/Utils.java | 15 +++-
.../manager/tableOps/bulkVer2/LoadFiles.java | 23 +++---
.../manager/tableOps/bulkVer2/PrepBulkImport.java | 42 ++++++++---
.../tableOps/bulkVer2/PrepBulkImportTest.java | 31 +++++---
.../org/apache/accumulo/tserver/ScanServer.java | 10 ++-
.../java/org/apache/accumulo/test/AmpleIT.java | 16 +++--
.../java/org/apache/accumulo/test/GCRunIT.java | 13 ++--
.../accumulo/test/ScanServerMetadataEntriesIT.java | 6 +-
.../test/compaction/ExternalCompaction4_IT.java | 33 +++++----
.../test/compaction/ExternalCompaction_2_IT.java | 8 +--
.../test/functional/GarbageCollectorTrashBase.java | 17 ++---
.../accumulo/test/functional/MetadataIT.java | 15 ++--
22 files changed, 335 insertions(+), 232 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 09e4174dd7,3b7bf91612..157df582ac
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@@ -1306,37 -1328,38 +1303,41 @@@ public class TableOperationsImpl extend
Text continueRow = null;
MapCounter<String> serverCounts = new MapCounter<>();
- for (TabletMetadata tablet : tablets) {
- total++;
- Location loc = tablet.getLocation();
- TabletHostingGoal goal = tablet.getHostingGoal();
-
- if ((expectedState == TableState.ONLINE
- && (goal == TabletHostingGoal.ALWAYS
- || (goal == TabletHostingGoal.ONDEMAND) &&
tablet.getHostingRequested())
- && (loc == null || loc.getType() == LocationType.FUTURE))
- || (expectedState == TableState.OFFLINE && loc != null)) {
- if (continueRow == null) {
- continueRow = tablet.getExtent().toMetaRow();
+ try (TabletsMetadata tablets =
TabletsMetadata.builder(context).scanMetadataTable()
- .overRange(range).fetch(LOCATION, PREV_ROW).build()) {
++ .overRange(range).fetch(HOSTING_GOAL, HOSTING_REQUESTED, LOCATION,
PREV_ROW).build()) {
+
+ for (TabletMetadata tablet : tablets) {
+ total++;
+ Location loc = tablet.getLocation();
++ TabletHostingGoal goal = tablet.getHostingGoal();
+
+ if ((expectedState == TableState.ONLINE
++ && (goal == TabletHostingGoal.ALWAYS
++ || (goal == TabletHostingGoal.ONDEMAND) &&
tablet.getHostingRequested())
+ && (loc == null || loc.getType() == LocationType.FUTURE))
+ || (expectedState == TableState.OFFLINE && loc != null)) {
+ if (continueRow == null) {
+ continueRow = tablet.getExtent().toMetaRow();
+ }
+ waitFor++;
+ lastRow = tablet.getExtent().toMetaRow();
+
+ if (loc != null) {
+ serverCounts.increment(loc.getHostPortSession(), 1);
+ }
}
- waitFor++;
- lastRow = tablet.getExtent().toMetaRow();
- if (loc != null) {
- serverCounts.increment(loc.getHostPortSession(), 1);
+ if (!tablet.getExtent().tableId().equals(tableId)) {
+ throw new AccumuloException(
+ "Saw unexpected table Id " + tableId + " " +
tablet.getExtent());
}
- }
- if (!tablet.getExtent().tableId().equals(tableId)) {
- throw new AccumuloException(
- "Saw unexpected table Id " + tableId + " " +
tablet.getExtent());
- }
+ if (lastExtent != null &&
!tablet.getExtent().isPreviousExtent(lastExtent)) {
+ holes++;
+ }
- if (lastExtent != null &&
!tablet.getExtent().isPreviousExtent(lastExtent)) {
- holes++;
+ lastExtent = tablet.getExtent();
}
-
- lastExtent = tablet.getExtent();
}
if (continueRow != null) {
@@@ -2098,9 -2060,11 +2099,11 @@@
@Override
public TimeType getTimeType(final String tableName) throws
TableNotFoundException {
TableId tableId = context.getTableId(tableName);
- Optional<TabletMetadata> tabletMetadata =
context.getAmple().readTablets().forTable(tableId)
- .fetch(TIME).checkConsistency().build().stream().findFirst();
-
+ Optional<TabletMetadata> tabletMetadata;
- try (TabletsMetadata tabletsMetadata =
context.getAmple().readTablets().forTable(tableId)
- .fetch(TabletMetadata.ColumnType.TIME).checkConsistency().build()) {
++ try (TabletsMetadata tabletsMetadata =
++
context.getAmple().readTablets().forTable(tableId).fetch(TIME).checkConsistency().build())
{
+ tabletMetadata = tabletsMetadata.stream().findFirst();
+ }
TabletMetadata timeData =
tabletMetadata.orElseThrow(() -> new IllegalStateException("Failed to
retrieve TimeType"));
return timeData.getTime().getType();
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
index 1d6fe1c6ee,02924116ec..86b8c2b71e
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
@@@ -313,19 -328,25 +313,20 @@@ class LoadFiles extends ManagerRepo
Text startRow = loadMapEntry.getKey().prevEndRow();
- Iterator<TabletMetadata> tabletIter =
-
TabletsMetadata.builder(manager.getContext()).forTable(tableId).overlapping(startRow,
null)
- .checkConsistency().fetch(PREV_ROW, LOCATION, LOADED,
TIME).build().iterator();
-
- Loader loader;
- if (bulkInfo.tableState == TableState.ONLINE) {
- loader = new OnlineLoader();
- } else {
- loader = new OfflineLoader();
- }
+ Loader loader = new Loader();
-
+ long t1;
loader.start(bulkDir, manager, tid, bulkInfo.setTime);
-
- long t1 = System.currentTimeMillis();
- while (lmi.hasNext()) {
- loadMapEntry = lmi.next();
- List<TabletMetadata> tablets =
findOverlappingTablets(loadMapEntry.getKey(), tabletIter);
- loader.load(tablets, loadMapEntry.getValue());
+ try (TabletsMetadata tabletsMetadata =
+
TabletsMetadata.builder(manager.getContext()).forTable(tableId).overlapping(startRow,
null)
- .checkConsistency().fetch(PREV_ROW, LOCATION, LOADED).build()) {
++ .checkConsistency().fetch(PREV_ROW, LOCATION, LOADED,
TIME).build()) {
+
+ t1 = System.currentTimeMillis();
+ while (lmi.hasNext()) {
+ loadMapEntry = lmi.next();
+ List<TabletMetadata> tablets =
+ findOverlappingTablets(loadMapEntry.getKey(),
tabletsMetadata.iterator());
+ loader.load(tablets, loadMapEntry.getValue());
+ }
}
long sleepTime = loader.finish();
diff --cc
test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java
index 4d4364e980,b52e7a41c7..43585cfffd
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java
@@@ -81,9 -89,13 +83,10 @@@ public class ExternalCompaction4_IT ext
client.tableOperations().attachIterator(table1, setting2,
EnumSet.of(IteratorScope.majc));
client.tableOperations().compact(table1, new
CompactionConfig().setWait(true));
- assertThrows(NoSuchElementException.class, () ->
ample.readTablets().forTable(tid)
- .fetch(ColumnType.FILES).build().iterator().next());
+ try (TabletsMetadata tm =
ample.readTablets().forTable(tid).fetch(ColumnType.FILES).build()) {
+ assertThrows(NoSuchElementException.class, () ->
tm.iterator().next());
+ }
assertEquals(0, client.createScanner(table1).stream().count());
- } finally {
- getCluster().getClusterControl().stopAllServers(ServerType.COMPACTOR);
-
getCluster().getClusterControl().stopAllServers(ServerType.COMPACTION_COORDINATOR);
}
}
@@@ -111,11 -124,11 +116,13 @@@
client.tableOperations().attachIterator(table1, setting,
EnumSet.of(IteratorScope.majc));
client.tableOperations().compact(table1, new
CompactionConfig().setWait(true));
- tms = ample.readTablets().forTable(tid).fetch(ColumnType.FILES,
ColumnType.ECOMP).build();
- tm = tms.iterator().next();
- assertEquals(1, tm.getFiles().size());
- // ensure the failed compactions did not leave anything in the metadata
table
- assertEquals(0, tm.getExternalCompactions().size());
- try (
- TabletsMetadata tms =
ample.readTablets().forTable(tid).fetch(ColumnType.FILES).build()) {
++ try (TabletsMetadata tms =
++ ample.readTablets().forTable(tid).fetch(ColumnType.FILES,
ColumnType.ECOMP).build()) {
+ TabletMetadata tm = tms.iterator().next();
+ assertEquals(1, tm.getFiles().size());
++ // ensure the failed compactions did not leave anything in the
metadata table
++ assertEquals(0, tm.getExternalCompactions().size());
+ }
ReadWriteIT.verify(client, 1000, 1, 1, 0, table1);
diff --cc
test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java
index 8a18b0bab3,3b902a5eab..6e2bc673fb
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java
@@@ -292,19 -333,14 +292,19 @@@ public class ExternalCompaction_2_IT ex
client.tableOperations().delete(table1);
- confirmCompactionCompleted(getCluster().getServerContext(), ecids,
- TCompactionState.CANCELLED);
+ LoggerFactory.getLogger(getClass()).debug("Table deleted.");
- try (TabletsMetadata tm =
getCluster().getServerContext().getAmple().readTablets()
- .forTable(tid).fetch(ColumnType.ECOMP).build()) {
+ confirmCompactionCompleted(ctx, ecids, TCompactionState.CANCELLED);
+
+ LoggerFactory.getLogger(getClass()).debug("Confirmed compaction
cancelled.");
+
- TabletsMetadata tm =
-
ctx.getAmple().readTablets().forTable(tid).fetch(ColumnType.PREV_ROW).build();
- assertEquals(0, tm.stream().count());
- tm.close();
++ try (TabletsMetadata tm =
++
ctx.getAmple().readTablets().forTable(tid).fetch(ColumnType.PREV_ROW).build()) {
+ assertEquals(0, tm.stream().count());
+ }
+ t.join();
+ assertNull(error.get());
}
}
