This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3f51b5e0ca Update MetaSplitIT to verify fenced files (#3931)
3f51b5e0ca is described below
commit 3f51b5e0caedae759eb6ba6a28a0ae8f7bc9d0f2
Author: Christopher L. Shannon <christopher.l.shan...@gmail.com>
AuthorDate: Tue Nov 7 08:45:39 2023 -0500
Update MetaSplitIT to verify fenced files (#3931)
This test already splits and merges the metadata table so the updates
here add some extra checks to verify ranged files have been created when
they should and that all data can still be scanned after ranged files
were created on merge.
This addresses part of #3766
Co-authored-by: Keith Turner <ktur...@apache.org>
---
.../java/org/apache/accumulo/test/MetaSplitIT.java | 62 ++++++++++++++++++++++
1 file changed, 62 insertions(+)
diff --git a/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
b/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
index 72b63d295f..0753463639 100644
--- a/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
@@ -18,24 +18,35 @@
*/
package org.apache.accumulo.test;
+import static org.apache.accumulo.test.util.FileMetadataUtil.countFencedFiles;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.AfterEach;
@@ -111,23 +122,49 @@ public class MetaSplitIT extends AccumuloClusterHarness {
@Test
public void testMetadataTableSplit() throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ // disable compactions
+ client.tableOperations().setProperty(MetadataTable.NAME,
Property.TABLE_MAJC_RATIO.getKey(),
+ "9999");
+
TableOperations opts = client.tableOperations();
for (int i = 1; i <= 10; i++) {
opts.create(Integer.toString(i));
}
try {
+ assertEquals(0, countFencedFiles(getServerContext(),
MetadataTable.NAME));
+ verifyMetadataTableScan(client);
opts.merge(MetadataTable.NAME, new Text("01"), new Text("02"));
checkMetadataSplits(1, opts);
+ verifyMetadataTableScan(client);
addSplits(opts, "4 5 6 7 8".split(" "));
checkMetadataSplits(6, opts);
+ verifyMetadataTableScan(client);
+
opts.merge(MetadataTable.NAME, new Text("6"), new Text("9"));
checkMetadataSplits(4, opts);
+ // Merging tablets should produce fenced files because of no-chop merge
+ assertTrue(countFencedFiles(getServerContext(), MetadataTable.NAME) >
0);
+ verifyMetadataTableScan(client);
+
addSplits(opts, "44 55 66 77 88".split(" "));
checkMetadataSplits(9, opts);
+ assertTrue(countFencedFiles(getServerContext(), MetadataTable.NAME) >
0);
+ verifyMetadataTableScan(client);
+
opts.merge(MetadataTable.NAME, new Text("5"), new Text("7"));
checkMetadataSplits(6, opts);
+ assertTrue(countFencedFiles(getServerContext(), MetadataTable.NAME) >
0);
+ verifyMetadataTableScan(client);
+
opts.merge(MetadataTable.NAME, null, null);
checkMetadataSplits(0, opts);
+ assertTrue(countFencedFiles(getServerContext(), MetadataTable.NAME) >
0);
+ verifyMetadataTableScan(client);
+
+ opts.compact(MetadataTable.NAME, new CompactionConfig());
+ // Should be no more fenced files after compaction
+ assertEquals(0, countFencedFiles(getServerContext(),
MetadataTable.NAME));
+ verifyMetadataTableScan(client);
} finally {
for (int i = 1; i <= 10; i++) {
opts.delete(Integer.toString(i));
@@ -136,6 +173,31 @@ public class MetaSplitIT extends AccumuloClusterHarness {
}
}
+ // Count the number of entries that can be read in the Metadata table
+ // This verifies all the entries can still be read after splits/merges
+ // when ranged files are used
+ private void verifyMetadataTableScan(AccumuloClient client) throws Exception
{
+ var tables = client.tableOperations().tableIdMap();
+ var expectedExtents = tables.entrySet().stream()
+ .filter(e ->
!e.getKey().startsWith("accumulo.")).map(Map.Entry::getValue).map(TableId::of)
+ .map(tid -> new KeyExtent(tid, null,
null)).collect(Collectors.toSet());
+ // Verify we have 10 tablets for metadata
+ assertEquals(10, expectedExtents.size());
+
+ // Scan each tablet to verify data exists
+ var ample = ((ClientContext) client).getAmple();
+ try (var tablets =
ample.readTablets().forLevel(Ample.DataLevel.USER).build()) {
+ for (var tablet : tablets) {
+ assertTrue(expectedExtents.remove(tablet.getExtent()));
+ assertNotNull(tablet.getDirName());
+ assertNotNull(tablet.getLocation());
+ }
+ }
+
+ // ensure all expected extents were seen
+ assertEquals(0, expectedExtents.size());
+ }
+
private static void checkMetadataSplits(int numSplits, TableOperations opts)
throws AccumuloSecurityException, TableNotFoundException,
AccumuloException,
InterruptedException {
