This is an automated email from the ASF dual-hosted git repository.
domgarguilo pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 165d169154 Close usages of TabletsMetadata (#4176)
165d169154 is described below
commit 165d1691543d11f636c5be22a9e1700608287d52
Author: Dom G <domgargu...@apache.org>
AuthorDate: Wed Jan 24 14:51:53 2024 -0500
Close usages of TabletsMetadata (#4176)
---
.../server/util/FindCompactionTmpFiles.java | 13 ++-
.../accumulo/server/util/FindOfflineTablets.java | 22 ++--
.../accumulo/server/util/MetadataTableUtil.java | 128 +++++++++++----------
.../coordinator/CompactionCoordinator.java | 10 +-
.../coordinator/DeadCompactionDetector.java | 13 ++-
.../manager/tableOps/goal/SetHostingGoal.java | 8 +-
.../commands/GetTabletHostingGoalCommand.java | 11 +-
.../shell/commands/ListTabletsCommand.java | 36 +++---
.../org/apache/accumulo/test/ComprehensiveIT.java | 19 +--
.../apache/accumulo/test/TableOperationsIT.java | 15 ++-
.../compaction/ExternalCompactionTestUtils.java | 19 +--
.../test/functional/AmpleConditionalWriterIT.java | 2 +-
.../accumulo/test/functional/CompactionIT.java | 14 ++-
.../test/functional/FindCompactionTmpFilesIT.java | 6 +-
.../test/functional/ManagerAssignmentIT.java | 8 +-
.../accumulo/test/functional/SplitMillionIT.java | 7 +-
.../functional/TabletResourceGroupBalanceIT.java | 29 ++---
17 files changed, 205 insertions(+), 155 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java
b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java
index aec3e775fb..eacfac6691 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java
@@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.volume.Volume;
@@ -111,11 +112,13 @@ public class FindCompactionTmpFiles {
// tmp files found on the filesystem. This must be done *after* gathering
the
// matches on the filesystem.
for (DataLevel level : DataLevel.values()) {
-
context.getAmple().readTablets().forLevel(level).fetch(ColumnType.ECOMP).build()
- .forEach(tm -> {
- tm.getExternalCompactions().values()
- .forEach(ecm ->
matches.remove(ecm.getCompactTmpName().getPath()));
- });
+ try (TabletsMetadata tabletsMetadata =
+
context.getAmple().readTablets().forLevel(level).fetch(ColumnType.ECOMP).build())
{
+ tabletsMetadata.forEach(tm -> {
+ tm.getExternalCompactions().values()
+ .forEach(ecm ->
matches.remove(ecm.getCompactTmpName().getPath()));
+ });
+ }
}
LOG.trace("Final set of compaction tmp files after removing active
compactions: {}", matches);
return matches;
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
b/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
index 6ab14c3c39..cc69b67ffa 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
@@ -29,6 +29,7 @@ import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletState;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.cli.ServerUtilOpts;
@@ -74,14 +75,14 @@ public class FindOfflineTablets {
tservers.startListeningForTabletServerChanges();
scanning.set(true);
- Iterator<TabletMetadata> zooScanner =
-
context.getAmple().readTablets().forLevel(DataLevel.ROOT).build().iterator();
-
int offline = 0;
- System.out.println("Scanning zookeeper");
- if ((offline = checkTablets(context, zooScanner, tservers)) > 0) {
- return offline;
+ try (TabletsMetadata tabletsMetadata =
+ context.getAmple().readTablets().forLevel(DataLevel.ROOT).build()) {
+ System.out.println("Scanning zookeeper");
+ if ((offline = checkTablets(context, tabletsMetadata.iterator(),
tservers)) > 0) {
+ return offline;
+ }
}
if (AccumuloTable.ROOT.tableName().equals(tableName)) {
@@ -89,10 +90,11 @@ public class FindOfflineTablets {
}
System.out.println("Scanning " + AccumuloTable.ROOT.tableName());
- Iterator<TabletMetadata> rootScanner =
-
context.getAmple().readTablets().forLevel(DataLevel.METADATA).build().iterator();
- if ((offline = checkTablets(context, rootScanner, tservers)) > 0) {
- return offline;
+ try (TabletsMetadata tabletsMetadata =
+ context.getAmple().readTablets().forLevel(DataLevel.METADATA).build())
{
+ if ((offline = checkTablets(context, tabletsMetadata.iterator(),
tservers)) > 0) {
+ return offline;
+ }
}
if (AccumuloTable.METADATA.tableName().equals(tableName)) {
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 3e9188db5a..dad8ac357f 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -276,7 +276,7 @@ public class MetadataTableUtil {
return m;
}
- private static Iterable<TabletMetadata> createCloneScanner(String
testTableName, TableId tableId,
+ private static TabletsMetadata createCloneScanner(String testTableName,
TableId tableId,
AccumuloClient client) {
String tableName;
@@ -301,14 +301,16 @@ public class MetadataTableUtil {
public static void initializeClone(String testTableName, TableId srcTableId,
TableId tableId,
AccumuloClient client, BatchWriter bw) throws MutationsRejectedException
{
- Iterator<TabletMetadata> ti = createCloneScanner(testTableName,
srcTableId, client).iterator();
+ try (TabletsMetadata cloneScanner = createCloneScanner(testTableName,
srcTableId, client)) {
+ Iterator<TabletMetadata> ti = cloneScanner.iterator();
- if (!ti.hasNext()) {
- throw new IllegalStateException(" table deleted during clone?
srcTableId = " + srcTableId);
- }
+ if (!ti.hasNext()) {
+ throw new IllegalStateException(" table deleted during clone?
srcTableId = " + srcTableId);
+ }
- while (ti.hasNext()) {
- bw.addMutation(createCloneMutation(srcTableId, tableId,
ti.next().getKeyValues()));
+ while (ti.hasNext()) {
+ bw.addMutation(createCloneMutation(srcTableId, tableId,
ti.next().getKeyValues()));
+ }
}
bw.flush();
@@ -324,90 +326,92 @@ public class MetadataTableUtil {
AccumuloClient client, BatchWriter bw)
throws TableNotFoundException, MutationsRejectedException {
- Iterator<TabletMetadata> srcIter =
- createCloneScanner(testTableName, srcTableId, client).iterator();
- Iterator<TabletMetadata> cloneIter =
- createCloneScanner(testTableName, tableId, client).iterator();
-
- if (!cloneIter.hasNext() || !srcIter.hasNext()) {
- throw new IllegalStateException(
- " table deleted during clone? srcTableId = " + srcTableId + "
tableId=" + tableId);
- }
-
int rewrites = 0;
- while (cloneIter.hasNext()) {
- TabletMetadata cloneTablet = cloneIter.next();
- Text cloneEndRow = cloneTablet.getEndRow();
- HashSet<StoredTabletFile> cloneFiles = new HashSet<>();
+ try (TabletsMetadata srcTM = createCloneScanner(testTableName, srcTableId,
client);
+ TabletsMetadata cloneTM = createCloneScanner(testTableName, tableId,
client)) {
+ Iterator<TabletMetadata> srcIter = srcTM.iterator();
+ Iterator<TabletMetadata> cloneIter = cloneTM.iterator();
- boolean cloneSuccessful = cloneTablet.getCloned() != null;
-
- if (!cloneSuccessful) {
- cloneFiles.addAll(cloneTablet.getFiles());
+ if (!cloneIter.hasNext() || !srcIter.hasNext()) {
+ throw new IllegalStateException(
+ " table deleted during clone? srcTableId = " + srcTableId + "
tableId=" + tableId);
}
- List<TabletMetadata> srcTablets = new ArrayList<>();
- TabletMetadata srcTablet = srcIter.next();
- srcTablets.add(srcTablet);
+ while (cloneIter.hasNext()) {
+ TabletMetadata cloneTablet = cloneIter.next();
+ Text cloneEndRow = cloneTablet.getEndRow();
+ HashSet<StoredTabletFile> cloneFiles = new HashSet<>();
- Text srcEndRow = srcTablet.getEndRow();
- int cmp = compareEndRows(cloneEndRow, srcEndRow);
- if (cmp < 0) {
- throw new TabletDeletedException(
- "Tablets deleted from src during clone : " + cloneEndRow + " " +
srcEndRow);
- }
+ boolean cloneSuccessful = cloneTablet.getCloned() != null;
- HashSet<StoredTabletFile> srcFiles = new HashSet<>();
- if (!cloneSuccessful) {
- srcFiles.addAll(srcTablet.getFiles());
- }
+ if (!cloneSuccessful) {
+ cloneFiles.addAll(cloneTablet.getFiles());
+ }
- while (cmp > 0) {
- srcTablet = srcIter.next();
+ List<TabletMetadata> srcTablets = new ArrayList<>();
+ TabletMetadata srcTablet = srcIter.next();
srcTablets.add(srcTablet);
- srcEndRow = srcTablet.getEndRow();
- cmp = compareEndRows(cloneEndRow, srcEndRow);
+
+ Text srcEndRow = srcTablet.getEndRow();
+ int cmp = compareEndRows(cloneEndRow, srcEndRow);
if (cmp < 0) {
throw new TabletDeletedException(
"Tablets deleted from src during clone : " + cloneEndRow + " " +
srcEndRow);
}
+ HashSet<StoredTabletFile> srcFiles = new HashSet<>();
if (!cloneSuccessful) {
srcFiles.addAll(srcTablet.getFiles());
}
- }
- if (cloneSuccessful) {
- continue;
- }
+ while (cmp > 0) {
+ srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+ srcEndRow = srcTablet.getEndRow();
+ cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0) {
+ throw new TabletDeletedException(
+ "Tablets deleted from src during clone : " + cloneEndRow + " "
+ srcEndRow);
+ }
- if (srcFiles.containsAll(cloneFiles)) {
- // write out marker that this tablet was successfully cloned
- Mutation m = new Mutation(cloneTablet.getExtent().toMetaRow());
- m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK"));
- bw.addMutation(m);
- } else {
- // delete existing cloned tablet entry
- Mutation m = new Mutation(cloneTablet.getExtent().toMetaRow());
+ if (!cloneSuccessful) {
+ srcFiles.addAll(srcTablet.getFiles());
+ }
+ }
- for (Entry<Key,Value> entry : cloneTablet.getKeyValues().entrySet()) {
- Key k = entry.getKey();
- m.putDelete(k.getColumnFamily(), k.getColumnQualifier(),
k.getTimestamp());
+ if (cloneSuccessful) {
+ continue;
}
- bw.addMutation(m);
+ if (srcFiles.containsAll(cloneFiles)) {
+ // write out marker that this tablet was successfully cloned
+ Mutation m = new Mutation(cloneTablet.getExtent().toMetaRow());
+ m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK"));
+ bw.addMutation(m);
+ } else {
+ // delete existing cloned tablet entry
+ Mutation m = new Mutation(cloneTablet.getExtent().toMetaRow());
- for (TabletMetadata st : srcTablets) {
- bw.addMutation(createCloneMutation(srcTableId, tableId,
st.getKeyValues()));
- }
+ for (Entry<Key,Value> entry : cloneTablet.getKeyValues().entrySet())
{
+ Key k = entry.getKey();
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(),
k.getTimestamp());
+ }
+
+ bw.addMutation(m);
- rewrites++;
+ for (TabletMetadata st : srcTablets) {
+ bw.addMutation(createCloneMutation(srcTableId, tableId,
st.getKeyValues()));
+ }
+
+ rewrites++;
+ }
}
}
bw.flush();
return rewrites;
+
}
public static void cloneTable(ServerContext context, TableId srcTableId,
TableId tableId)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index 283d2c1327..43477c46b3 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -82,6 +82,7 @@ import
org.apache.accumulo.core.metadata.schema.CompactionMetadata;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.metrics.MetricsProducer;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
@@ -841,9 +842,12 @@ public class CompactionCoordinator
}
protected Set<ExternalCompactionId> readExternalCompactionIds() {
- return this.ctx.getAmple().readTablets().forLevel(Ample.DataLevel.USER)
- .filter(new
HasExternalCompactionsFilter()).fetch(ECOMP).build().stream()
- .flatMap(tm ->
tm.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
+ try (TabletsMetadata tabletsMetadata =
+ this.ctx.getAmple().readTablets().forLevel(Ample.DataLevel.USER)
+ .filter(new HasExternalCompactionsFilter()).fetch(ECOMP).build()) {
+ return tabletsMetadata.stream().flatMap(tm ->
tm.getExternalCompactions().keySet().stream())
+ .collect(Collectors.toSet());
+ }
}
/**
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java
index 2fabf68cda..1de507f2a3 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java
@@ -36,6 +36,7 @@ import
org.apache.accumulo.core.iterators.user.HasExternalCompactionsFilter;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.server.ServerContext;
@@ -78,13 +79,15 @@ public class DeadCompactionDetector {
Map<ExternalCompactionId,KeyExtent> tabletCompactions = new HashMap<>();
//
// find what external compactions tablets think are running
- context.getAmple().readTablets().forLevel(DataLevel.USER)
+ try (TabletsMetadata tabletsMetadata =
context.getAmple().readTablets().forLevel(DataLevel.USER)
.filter(new HasExternalCompactionsFilter()).fetch(ColumnType.ECOMP,
ColumnType.PREV_ROW)
- .build().forEach(tm -> {
- tm.getExternalCompactions().keySet().forEach(ecid -> {
- tabletCompactions.put(ecid, tm.getExtent());
- });
+ .build()) {
+ tabletsMetadata.forEach(tm -> {
+ tm.getExternalCompactions().keySet().forEach(ecid -> {
+ tabletCompactions.put(ecid, tm.getExtent());
});
+ });
+ }
if (tabletCompactions.isEmpty()) {
// Clear out dead compactions, tservers don't think anything is running
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/goal/SetHostingGoal.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/goal/SetHostingGoal.java
index a508324f76..cb5f5c79cc 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/goal/SetHostingGoal.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/goal/SetHostingGoal.java
@@ -78,10 +78,10 @@ public class SetHostingGoal extends ManagerRepo {
// row is always inclusive.
final Text scanRangeStart = (range.getStartKey() == null) ? null :
range.getStartKey().getRow();
- TabletsMetadata m =
manager.getContext().getAmple().readTablets().forTable(tableId)
- .overlapping(scanRangeStart, true, null).build();
-
- try (TabletsMutator mutator =
manager.getContext().getAmple().mutateTablets()) {
+ try (
+ TabletsMetadata m =
manager.getContext().getAmple().readTablets().forTable(tableId)
+ .overlapping(scanRangeStart, true, null).build();
+ TabletsMutator mutator =
manager.getContext().getAmple().mutateTablets()) {
for (TabletMetadata tm : m) {
final KeyExtent tabletExtent = tm.getExtent();
LOG.trace("Evaluating tablet {} against range {}", tabletExtent,
range);
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/GetTabletHostingGoalCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/GetTabletHostingGoalCommand.java
index 1588e2924c..9353459b5b 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/GetTabletHostingGoalCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/GetTabletHostingGoalCommand.java
@@ -18,6 +18,9 @@
*/
package org.apache.accumulo.shell.commands;
+import java.util.stream.Stream;
+
+import org.apache.accumulo.core.client.admin.TabletInformation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.shell.Shell;
import org.apache.commons.cli.CommandLine;
@@ -46,9 +49,11 @@ public class GetTabletHostingGoalCommand extends
TableOperation {
protected void doTableOp(Shell shellState, String tableName) throws
Exception {
shellState.getWriter().println("TABLE: " + tableName);
shellState.getWriter().println("TABLET ID HOSTING GOAL");
-
shellState.getAccumuloClient().tableOperations().getTabletInformation(tableName,
range)
- .forEach(p -> shellState.getWriter()
- .println(String.format("%-10s %s", p.getTabletId(),
p.getHostingGoal())));
+ try (Stream<TabletInformation> tabletInformation =
+
shellState.getAccumuloClient().tableOperations().getTabletInformation(tableName,
range)) {
+ tabletInformation.forEach(p -> shellState.getWriter()
+ .println(String.format("%-10s %s", p.getTabletId(),
p.getHostingGoal())));
+ }
}
@Override
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java
index abc6db9f40..ad36eb665f 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java
@@ -24,8 +24,9 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.accumulo.core.client.NamespaceNotFoundException;
import org.apache.accumulo.core.client.admin.TableOperations;
@@ -81,21 +82,24 @@ public class ListTabletsCommand extends Command {
String name = tableInfo.name;
lines.add("TABLE: " + name);
- List<TabletInformation> tabletsList =
shellState.getContext().tableOperations()
- .getTabletInformation(name, new
Range()).collect(Collectors.toList());
- for (int i = 0; i < tabletsList.size(); i++) {
- TabletInformation tabletInfo = tabletsList.get(i);
- lines.add(String.format("%-4d %-15s %-5d %-5s %-9s %-9s %-10s %-30s
%-5s %-20s %-20s %-10s",
- i + 1, tabletInfo.getTabletDir(), tabletInfo.getNumFiles(),
tabletInfo.getNumWalLogs(),
- getEstimatedEntries(tabletInfo.getEstimatedEntries(),
humanReadable),
- getEstimatedSize(tabletInfo.getEstimatedSize(), humanReadable),
- tabletInfo.getTabletState(),
tabletInfo.getLocation().orElse("None"),
- tabletInfo.getTabletId().getTable(),
- tabletInfo.getTabletId().getPrevEndRow() == null ? "-INF"
- : tabletInfo.getTabletId().getPrevEndRow().toString(),
- tabletInfo.getTabletId().getEndRow() == null ? "+INF"
- : tabletInfo.getTabletId().getEndRow().toString(),
- tabletInfo.getHostingGoal()));
+ try (Stream<TabletInformation> tabletInfoStream =
+ shellState.getContext().tableOperations().getTabletInformation(name,
new Range())) {
+ final AtomicInteger counter = new AtomicInteger(1);
+ tabletInfoStream.forEach(tabletInfo -> {
+ int i = counter.getAndIncrement();
+ lines.add(String.format(
+ "%-4d %-15s %-5d %-5s %-9s %-9s %-10s %-30s %-5s %-20s %-20s
%-10s", i,
+ tabletInfo.getTabletDir(), tabletInfo.getNumFiles(),
tabletInfo.getNumWalLogs(),
+ getEstimatedEntries(tabletInfo.getEstimatedEntries(),
humanReadable),
+ getEstimatedSize(tabletInfo.getEstimatedSize(), humanReadable),
+ tabletInfo.getTabletState(),
tabletInfo.getLocation().orElse("None"),
+ tabletInfo.getTabletId().getTable(),
+ tabletInfo.getTabletId().getPrevEndRow() == null ? "-INF"
+ : tabletInfo.getTabletId().getPrevEndRow().toString(),
+ tabletInfo.getTabletId().getEndRow() == null ? "+INF"
+ : tabletInfo.getTabletId().getEndRow().toString(),
+ tabletInfo.getHostingGoal()));
+ });
}
}
diff --git a/test/src/main/java/org/apache/accumulo/test/ComprehensiveIT.java
b/test/src/main/java/org/apache/accumulo/test/ComprehensiveIT.java
index fb7215216b..4afe1e1560 100644
--- a/test/src/main/java/org/apache/accumulo/test/ComprehensiveIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ComprehensiveIT.java
@@ -44,6 +44,7 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Predicate;
+import java.util.stream.Stream;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
@@ -62,6 +63,7 @@ import
org.apache.accumulo.core.client.admin.CloneConfiguration;
import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TabletHostingGoal;
+import org.apache.accumulo.core.client.admin.TabletInformation;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.client.rfile.RFile;
import org.apache.accumulo.core.client.sample.Sampler;
@@ -914,13 +916,16 @@ public class ComprehensiveIT extends
SharedMiniClusterBase {
assertEquals(iterSetting,
client.tableOperations().getIteratorSetting(table, "fam9",
IteratorUtil.IteratorScope.scan));
- client.tableOperations().getTabletInformation(table, new
Range()).forEach(tabletInformation -> {
- if (tabletInformation.getTabletId().getEndRow() == null) {
- assertEquals(TabletHostingGoal.ALWAYS,
tabletInformation.getHostingGoal());
- } else {
- assertEquals(TabletHostingGoal.ONDEMAND,
tabletInformation.getHostingGoal());
- }
- });
+ try (Stream<TabletInformation> tabletInfo =
+ client.tableOperations().getTabletInformation(table, new Range())) {
+ tabletInfo.forEach(tabletInformation -> {
+ if (tabletInformation.getTabletId().getEndRow() == null) {
+ assertEquals(TabletHostingGoal.ALWAYS,
tabletInformation.getHostingGoal());
+ } else {
+ assertEquals(TabletHostingGoal.ONDEMAND,
tabletInformation.getHostingGoal());
+ }
+ });
+ }
verifyData(client, table, Authorizations.EMPTY,
generateKeys(0, 100, tr -> tr.fam != 9 && tr.vis.isEmpty()));
diff --git a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
index 3ce5f2d46e..bd1d71836d 100644
--- a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
@@ -40,6 +40,7 @@ import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
@@ -688,12 +689,14 @@ public class TableOperationsIT extends
AccumuloClusterHarness {
private void verifyTabletGoals(String tableName, Range range,
List<HostingGoalForTablet> expectedGoals) throws TableNotFoundException {
- List<TabletInformation> tabletInfo = accumuloClient.tableOperations()
- .getTabletInformation(tableName, range).collect(Collectors.toList());
- assertEquals(expectedGoals.size(), tabletInfo.size());
- for (var i = 0; i < expectedGoals.size(); i++) {
- assertEquals(expectedGoals.get(i).getTabletId(),
tabletInfo.get(i).getTabletId());
- assertEquals(expectedGoals.get(i).getHostingGoal(),
tabletInfo.get(i).getHostingGoal());
+ try (Stream<TabletInformation> tabletInformation =
+ accumuloClient.tableOperations().getTabletInformation(tableName,
range)) {
+ List<TabletInformation> tabletInfo =
tabletInformation.collect(Collectors.toList());
+ assertEquals(expectedGoals.size(), tabletInfo.size());
+ for (var i = 0; i < expectedGoals.size(); i++) {
+ assertEquals(expectedGoals.get(i).getTabletId(),
tabletInfo.get(i).getTabletId());
+ assertEquals(expectedGoals.get(i).getHostingGoal(),
tabletInfo.get(i).getHostingGoal());
+ }
}
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java
index 3dbdc228b4..b91dee97cf 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java
@@ -369,17 +369,18 @@ public class ExternalCompactionTestUtils {
public static void assertNoCompactionMetadata(ServerContext ctx, String
tableName) {
var tableId =
TableId.of(ctx.tableOperations().tableIdMap().get(tableName));
- var tabletsMetadata =
ctx.getAmple().readTablets().forTable(tableId).build();
+ try (var tabletsMetadata =
ctx.getAmple().readTablets().forTable(tableId).build()) {
- int count = 0;
+ int count = 0;
- for (var tabletMetadata : tabletsMetadata) {
- assertEquals(Set.of(), tabletMetadata.getCompacted());
- assertNull(tabletMetadata.getSelectedFiles());
- assertEquals(Set.of(), tabletMetadata.getExternalCompactions().keySet());
- count++;
- }
+ for (var tabletMetadata : tabletsMetadata) {
+ assertEquals(Set.of(), tabletMetadata.getCompacted());
+ assertNull(tabletMetadata.getSelectedFiles());
+ assertEquals(Set.of(),
tabletMetadata.getExternalCompactions().keySet());
+ count++;
+ }
- assertTrue(count > 0);
+ assertTrue(count > 0);
+ }
}
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
index f5eda341b3..4ece846fd5 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
@@ -1061,7 +1061,7 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
// if we only fetch some columns needed by the filter, we should get an
exception
TabletsMetadata.Options options =
context.getAmple().readTablets().forTable(tid).fetch(FLUSH_ID).filter(filter);
- var ise = assertThrows(IllegalStateException.class, options::build);
+ var ise = assertThrows(IllegalStateException.class, () ->
options.build().close());
String expectedMsg = String.format("%s needs cols %s however only %s
were fetched",
TestTabletMetadataFilter.class.getSimpleName(), filter.getColumns(),
Set.of(FLUSH_ID));
assertTrue(ise.getMessage().contains(expectedMsg));
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index a6146085da..b734819200 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@ -538,8 +538,11 @@ public class CompactionIT extends AccumuloClusterHarness {
// creating a user table should cause a write to the metadata table
c.tableOperations().create(tableNames[0]);
- var mfiles1 = getServerContext().getAmple().readTablets()
-
.forTable(AccumuloTable.METADATA.tableId()).build().iterator().next().getFiles();
+ Set<StoredTabletFile> mfiles1;
+ try (TabletsMetadata tabletsMetadata =
getServerContext().getAmple().readTablets()
+ .forTable(AccumuloTable.METADATA.tableId()).build()) {
+ mfiles1 = tabletsMetadata.iterator().next().getFiles();
+ }
var rootFiles1 =
getServerContext().getAmple().readTablet(RootTable.EXTENT).getFiles();
log.debug("mfiles1 {}",
@@ -567,8 +570,11 @@ public class CompactionIT extends AccumuloClusterHarness {
// eventually equal one.
Wait.waitFor(() -> {
- var mfiles2 = getServerContext().getAmple().readTablets()
-
.forTable(AccumuloTable.METADATA.tableId()).build().iterator().next().getFiles();
+ Set<StoredTabletFile> mfiles2;
+ try (TabletsMetadata tabletsMetadata =
getServerContext().getAmple().readTablets()
+ .forTable(AccumuloTable.METADATA.tableId()).build()) {
+ mfiles2 = tabletsMetadata.iterator().next().getFiles();
+ }
log.debug("mfiles2 {}",
mfiles2.stream().map(StoredTabletFile::getFileName).collect(toList()));
return mfiles2.size() == 1 && !mfiles2.equals(mfiles1);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT.java
index 7946ea0a12..af4bcefe5c 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT.java
@@ -59,8 +59,10 @@ public class FindCompactionTmpFilesIT extends
SharedMiniClusterBase {
private Set<Path> generateTmpFilePaths(ServerContext context, TableId tid,
Path tabletDir,
int numFiles) {
final Set<Path> paths = new HashSet<>(numFiles);
- final TabletsMetadata tms =
context.getAmple().readTablets().forTable(tid).build();
- final TabletMetadata tm = tms.iterator().next();
+ final TabletMetadata tm;
+ try (TabletsMetadata tms =
context.getAmple().readTablets().forTable(tid).build()) {
+ tm = tms.iterator().next();
+ }
for (int i = 0; i < numFiles; i++) {
ReferencedTabletFile rtf =
TabletNameGenerator.getNextDataFilenameForMajc(false, context, tm,
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
index d1e79332a2..87076c008e 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
@@ -70,6 +70,7 @@ import
org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
import org.apache.accumulo.core.metadata.schema.TabletOperationId;
import org.apache.accumulo.core.metadata.schema.TabletOperationType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.spi.ondemand.DefaultOnDemandTabletUnloader;
@@ -461,8 +462,11 @@ public class ManagerAssignmentIT extends
SharedMiniClusterBase {
}
public static long countTabletsWithLocation(AccumuloClient c, TableId
tableId) {
- return
getAmple(c).readTablets().forTable(tableId).fetch(TabletMetadata.ColumnType.LOCATION)
- .build().stream().filter(tabletMetadata ->
tabletMetadata.getLocation() != null).count();
+ try (TabletsMetadata tabletsMetadata =
getAmple(c).readTablets().forTable(tableId)
+ .fetch(TabletMetadata.ColumnType.LOCATION).build()) {
+ return tabletsMetadata.stream().filter(tabletMetadata ->
tabletMetadata.getLocation() != null)
+ .count();
+ }
}
public static List<TabletStats> getTabletStats(AccumuloClient c, String
tableId)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/SplitMillionIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/SplitMillionIT.java
index 3ed25d330f..585428bb32 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SplitMillionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitMillionIT.java
@@ -117,8 +117,12 @@ public class SplitMillionIT extends AccumuloClusterHarness
{
log.info("Row: {} scan1: {}ms write: {}ms scan2: {}ms", row, t2 - t1,
t3 - t2, t4 - t3);
}
+ long count;
long t1 = System.currentTimeMillis();
- long count = c.tableOperations().getTabletInformation(tableName, new
Range()).count();
+ try (var tabletInformation =
+ c.tableOperations().getTabletInformation(tableName, new Range())) {
+ count = tabletInformation.count();
+ }
long t2 = System.currentTimeMillis();
assertEquals(1_000_000, count);
log.info("Time to scan all tablets information : {}ms", t2 - t1);
@@ -128,7 +132,6 @@ public class SplitMillionIT extends AccumuloClusterHarness {
c.tableOperations().compact(tableName,
new
CompactionConfig().setIterators(List.of(iterSetting)).setWait(true).setFlush(true));
t2 = System.currentTimeMillis();
- assertEquals(1_000_000, count);
log.info("Time to compact all tablets : {}ms", t2 - t1);
var expected = Map.of("y", "900", "z", "300");
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletResourceGroupBalanceIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletResourceGroupBalanceIT.java
index 1e43027ab6..785cae2a94 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletResourceGroupBalanceIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletResourceGroupBalanceIT.java
@@ -56,6 +56,7 @@ import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.minicluster.ServerType;
@@ -123,6 +124,13 @@ public class TabletResourceGroupBalanceIT extends
SharedMiniClusterBase {
}
+ private static List<TabletMetadata> getLocations(Ample ample, String
tableId) {
+ try (TabletsMetadata tabletsMetadata =
ample.readTablets().forTable(TableId.of(tableId))
+ .fetch(TabletMetadata.ColumnType.LOCATION).build()) {
+ return tabletsMetadata.stream().collect(Collectors.toList());
+ }
+ }
+
@Test
public void testBalancerWithResourceGroups() throws Exception {
@@ -155,8 +163,7 @@ public class TabletResourceGroupBalanceIT extends
SharedMiniClusterBase {
// Check table names[0]
String tableId = client.tableOperations().tableIdMap().get(names[0]);
- List<TabletMetadata> locations =
ample.readTablets().forTable(TableId.of(tableId))
-
.fetch(TabletMetadata.ColumnType.LOCATION).build().stream().collect(Collectors.toList());
+ List<TabletMetadata> locations = getLocations(ample, tableId);
assertEquals(26, locations.size());
Location l1 = locations.get(0).getLocation();
@@ -165,8 +172,7 @@ public class TabletResourceGroupBalanceIT extends
SharedMiniClusterBase {
// Check table names[1]
tableId = client.tableOperations().tableIdMap().get(names[1]);
- locations = ample.readTablets().forTable(TableId.of(tableId))
-
.fetch(TabletMetadata.ColumnType.LOCATION).build().stream().collect(Collectors.toList());
+ locations = getLocations(ample, tableId);
assertEquals(26, locations.size());
Location l2 = locations.get(0).getLocation();
@@ -301,8 +307,7 @@ public class TabletResourceGroupBalanceIT extends
SharedMiniClusterBase {
// Validate that all of the tables tablets are on the same tserver and that
// the tserver is in the default resource group
- List<TabletMetadata> locations =
ample.readTablets().forTable(TableId.of(tableId))
-
.fetch(TabletMetadata.ColumnType.LOCATION).build().stream().collect(Collectors.toList());
+ List<TabletMetadata> locations = getLocations(ample, tableId);
assertEquals(numExpectedSplits, locations.size());
Location l1 = locations.get(0).getLocation();
assertEquals("default", tserverGroups.get(l1.getHostAndPort().toString()));
@@ -311,16 +316,14 @@ public class TabletResourceGroupBalanceIT extends
SharedMiniClusterBase {
// change the resource group property for the table
client.tableOperations().setProperty(tableName,
"table.custom.assignment.group", "GROUP1");
- locations = ample.readTablets().forTable(TableId.of(tableId))
-
.fetch(TabletMetadata.ColumnType.LOCATION).build().stream().collect(Collectors.toList());
+ locations = getLocations(ample, tableId);
// wait for GROUP1 to show up in the list of locations as the current
location
while ((locations == null || locations.isEmpty() || locations.size() !=
numExpectedSplits
|| locations.get(0).getLocation() == null
|| locations.get(0).getLocation().getType() == LocationType.FUTURE)
|| (locations.get(0).getLocation().getType() == LocationType.CURRENT
&& !tserverGroups
.get(locations.get(0).getLocation().getHostAndPort().toString()).equals("GROUP1")))
{
- locations = ample.readTablets().forTable(TableId.of(tableId))
-
.fetch(TabletMetadata.ColumnType.LOCATION).build().stream().collect(Collectors.toList());
+ locations = getLocations(ample, tableId);
}
Location group1Location = locations.get(0).getLocation();
assertTrue(tserverGroups.get(group1Location.getHostAndPort().toString()).equals("GROUP1"));
@@ -328,11 +331,9 @@ public class TabletResourceGroupBalanceIT extends
SharedMiniClusterBase {
client.instanceOperations().waitForBalance();
// validate that all tablets have the same location as the first tablet
- locations = ample.readTablets().forTable(TableId.of(tableId))
-
.fetch(TabletMetadata.ColumnType.LOCATION).build().stream().collect(Collectors.toList());
+ locations = getLocations(ample, tableId);
while (locations == null || locations.isEmpty() || locations.size() !=
numExpectedSplits) {
- locations = ample.readTablets().forTable(TableId.of(tableId))
-
.fetch(TabletMetadata.ColumnType.LOCATION).build().stream().collect(Collectors.toList());
+ locations = getLocations(ample, tableId);
}
if (locations.stream().map(TabletMetadata::getLocation)
.allMatch((l) -> group1Location.equals(l))) {
