This is an automated email from the ASF dual-hosted git repository.
krathbun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 34416a1b67c718221845745b37800f4071efd3b4
Merge: 17646f77eb aa75da3d27
Author: Kevin Rathbun <kevinrr...@gmail.com>
AuthorDate: Fri Apr 18 10:46:10 2025 -0400
Merge branch '2.1'
.../accumulo/server/util/TableDiskUsage.java | 79 ++++---
.../accumulo/server/util/TableDiskUsageTest.java | 227 +++++++++++----------
2 files changed, 157 insertions(+), 149 deletions(-)
diff --cc
server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java
index 009fcc1cf0,5b9ee7fed2..4b71657969
---
a/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java
@@@ -21,7 -21,10 +21,9 @@@ package org.apache.accumulo.server.util
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
+ import java.util.ArrayList;
-import java.util.EnumSet;
import java.util.HashMap;
+ import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@@ -32,19 -36,19 +34,15 @@@ import java.util.TreeSet
import java.util.stream.Collectors;
import org.apache.accumulo.core.Constants;
- import org.apache.accumulo.core.client.Scanner;
- import org.apache.accumulo.core.data.Key;
+ import org.apache.accumulo.core.clientImpl.ClientContext;
-import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.TableId;
--import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.AccumuloTable;
- import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+ import org.apache.accumulo.core.metadata.StoredTabletFile;
+ import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
--import org.apache.accumulo.core.metadata.schema.MetadataSchema;
- import org.apache.accumulo.core.security.Authorizations;
- import org.apache.accumulo.server.ServerContext;
- import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.io.Text;
+ import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+ import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
-import org.apache.hadoop.io.Text;
import org.easymock.EasyMock;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@@ -131,25 -140,21 +134,22 @@@ public class TableDiskUsageTest
@Test
public void testMetadataTable() throws Exception {
- final ServerContext client = EasyMock.createMock(ServerContext.class);
- final Scanner scanner = EasyMock.createMock(Scanner.class);
-
- // Expect root table instead to be scanned
- EasyMock.expect(client.createScanner(AccumuloTable.ROOT.tableName(),
Authorizations.EMPTY))
- .andReturn(scanner);
+ final ClientContext client = EasyMock.createMock(ClientContext.class);
EasyMock.expect(client.getTableIdToNameMap()).andReturn(tableIdToNameMap);
- final TabletsMetadata mockTabletsMetadata = mockTabletsMetadata(client,
MetadataTable.ID);
++ final TabletsMetadata mockTabletsMetadata =
++ mockTabletsMetadata(client, AccumuloTable.METADATA.tableId());
- Map<Key,Value> tableEntries = new HashMap<>();
- appendFileMetadata(tableEntries,
getTabletFile(AccumuloTable.METADATA.tableId(),
+ List<TabletMetadata> realTabletsMetadata = new ArrayList<>();
- appendFileMetadata(realTabletsMetadata,
- getTabletFile(volume1, MetadataTable.ID, MetadataTable.NAME,
"C0001.rf"), 1024);
++ appendFileMetadata(realTabletsMetadata, getTabletFile(volume1,
AccumuloTable.METADATA.tableId(),
+ AccumuloTable.METADATA.tableName(), "C0001.rf"), 1024);
- mockTableScan(scanner, tableEntries, AccumuloTable.METADATA.tableId());
+ mockTabletsMetadataIter(mockTabletsMetadata,
realTabletsMetadata.iterator());
- EasyMock.replay(client, scanner);
+ EasyMock.replay(client, mockTabletsMetadata);
Map<SortedSet<String>,Long> result =
- TableDiskUsage.getDiskUsage(tableSet(MetadataTable.ID), client);
+
TableDiskUsage.getDiskUsage(tableSet(AccumuloTable.METADATA.tableId()), client);
- assertEquals(1024, getTotalUsage(result, MetadataTable.ID));
+ assertEquals(1024, getTotalUsage(result,
AccumuloTable.METADATA.tableId()));
assertEquals(1, result.size());
Map.Entry<SortedSet<String>,Long> firstResult =
result.entrySet().stream().findFirst().orElseThrow();
@@@ -277,37 -294,46 +289,40 @@@
return tableIdToNameMap.get(tableId);
}
- private static void appendFileMetadata(Map<Key,Value> tableEntries,
ReferencedTabletFile file,
- long size) {
- tableEntries.put(new Key(new Text(file.getTableId() + "<"),
- MetadataSchema.TabletsSection.DataFileColumnFamily.NAME,
file.insert().getMetadataText()),
- new DataFileValue(size, 1).encodeAsValue());
+ private static void appendFileMetadata(List<TabletMetadata>
realTabletsMetadata,
+ StoredTabletFile file, long size) {
- Key key = new Key(new Text(file.getTableId() + "<"),
- MetadataSchema.TabletsSection.DataFileColumnFamily.NAME,
file.getMetaInsertText());
- Value val = new DataFileValue(size, 1).encodeAsValue();
- SortedMap<Key,Value> map = new TreeMap<>();
- map.put(key, val);
-
- TabletMetadata tm = TabletMetadata.convertRow(map.entrySet().iterator(),
- EnumSet.of(TabletMetadata.ColumnType.FILES), true);
++ TabletMetadata tm = TabletMetadata.builder(new
KeyExtent(file.getTableId(), null, null))
++ .putFile(file, new DataFileValue(size, 1)).build();
+ realTabletsMetadata.add(tm);
}
- private static ReferencedTabletFile getTabletFile(String volume, TableId
tableId, String tablet,
+ private static StoredTabletFile getTabletFile(String volume, TableId
tableId, String tablet,
String fileName) {
- return new ReferencedTabletFile(new Path(
- return new StoredTabletFile(
- volume + Constants.HDFS_TABLES_DIR + "/" + tableId + "/" + tablet +
"/" + fileName);
++ return new StoredTabletFile(StoredTabletFile.serialize(
+ volume + Constants.HDFS_TABLES_DIR + "/" + tableId + "/" + tablet +
"/" + fileName));
}
- private static ReferencedTabletFile getTabletFile(TableId tableId, String
tablet,
- String fileName) {
- return getTabletFile(volume1, tableId, tablet, fileName);
- }
-
- private void mockScan(ServerContext client, Scanner scanner, int times)
throws Exception {
- EasyMock.expect(client.createScanner(AccumuloTable.METADATA.tableName(),
Authorizations.EMPTY))
- .andReturn(scanner).times(times);
- EasyMock.expect(client.getTableIdToNameMap()).andReturn(tableIdToNameMap);
+ private TabletsMetadata mockTabletsMetadata(ClientContext client, TableId
tableId) {
+ final Ample ample = EasyMock.createMock(Ample.class);
+ final TabletsMetadata.TableOptions tableOptions =
+ EasyMock.createMock(TabletsMetadata.TableOptions.class);
+ final TabletsMetadata.TableRangeOptions tableRangeOptions =
+ EasyMock.createMock(TabletsMetadata.TableRangeOptions.class);
+ final TabletsMetadata.Options options =
EasyMock.createMock(TabletsMetadata.Options.class);
+ final TabletsMetadata tabletsMetadata =
EasyMock.createMock(TabletsMetadata.class);
+ EasyMock.expect(client.getAmple()).andReturn(ample);
+ EasyMock.expect(ample.readTablets()).andReturn(tableOptions);
+
EasyMock.expect(tableOptions.forTable(tableId)).andReturn(tableRangeOptions);
+
EasyMock.expect(tableRangeOptions.fetch(TabletMetadata.ColumnType.FILES)).andReturn(options);
+ EasyMock.expect(options.build()).andReturn(tabletsMetadata);
+ EasyMock.replay(ample, tableOptions, tableRangeOptions, options);
+ return tabletsMetadata;
}
- private void mockTableScan(Scanner scanner, Map<Key,Value> tableEntries,
TableId tableId) {
-
scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
- EasyMock.expectLastCall().once();
- scanner.setRange(new KeyExtent(tableId, null, null).toMetaRange());
- EasyMock.expectLastCall().once();
-
EasyMock.expect(scanner.iterator()).andReturn(tableEntries.entrySet().iterator());
- scanner.close();
- EasyMock.expectLastCall().once();
+ private void mockTabletsMetadataIter(TabletsMetadata tabletsMetadata,
+ Iterator<TabletMetadata> realTabletsMetadata) {
+
EasyMock.expect(tabletsMetadata.iterator()).andReturn(realTabletsMetadata);
+ tabletsMetadata.close();
+ EasyMock.expectLastCall().andAnswer(() -> null);
}
}
