This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit d7fcb8493ab3be238a4bb567eed7552aade802aa
Merge: 7139931de4 53683e4701
Author: Christopher L. Shannon <cshan...@apache.org>
AuthorDate: Sat Jun 1 15:57:26 2024 -0400
Merge branch 'main' into elasticity
.../apache/accumulo/core/client/rfile/RFile.java | 4 +-
.../apache/accumulo/core/dataImpl/KeyExtent.java | 61 ++++++++++++++++
.../org/apache/accumulo/core/file/rfile/RFile.java | 8 +-
.../accumulo/core/metadata/AbstractTabletFile.java | 48 ++----------
.../accumulo/core/metadata/StoredTabletFile.java | 7 +-
.../apache/accumulo/core/util/RowRangeUtil.java | 85 ++++++++++++++++++++++
.../core/client/rfile/RFileClientTest.java | 11 +--
.../apache/accumulo/core/data/KeyExtentTest.java | 43 +++++++++++
.../core/file/rfile/AbstractRFileTest.java | 37 +++++++++-
.../accumulo/core/file/rfile/FencedRFileTest.java | 72 +++++++++++++++++-
.../apache/accumulo/core/file/rfile/RFileTest.java | 33 +++++++++
.../metadata/schema/ReferencedTabletFileTest.java | 22 ++++--
.../core/metadata/schema/TabletMetadataTest.java | 12 +--
.../constraints/MetadataConstraintsTest.java | 15 ++--
.../java/org/apache/accumulo/test/CloneIT.java | 3 +-
.../java/org/apache/accumulo/test/VolumeIT.java | 7 +-
16 files changed, 389 insertions(+), 79 deletions(-)
diff --cc
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index 06571407e3,8c3d84de2c..d22925afc3
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -431,142 -408,10 +431,142 @@@ public class TabletMetadataTest
assertEquals(1, tm2.getScans().size());
assertThrows(UnsupportedOperationException.class, () ->
tm2.getScans().add(stf));
assertEquals(1, tm2.getLoaded().size());
- assertThrows(UnsupportedOperationException.class, () ->
tm2.getLoaded().put(stf, 0L));
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm2.getLoaded().put(stf, FateId.from(FateInstanceType.USER,
UUID.randomUUID())));
assertEquals(1, tm2.getKeyValues().size());
assertThrows(UnsupportedOperationException.class, () ->
tm2.getKeyValues().remove(null));
+ assertEquals(1, tm2.getCompacted().size());
+ assertThrows(UnsupportedOperationException.class,
+ () -> tm2.getCompacted().add(FateId.from(FateInstanceType.USER,
UUID.randomUUID())));
+ assertEquals(1, tm2.getUserCompactionsRequested().size());
+ assertThrows(UnsupportedOperationException.class, () ->
tm2.getUserCompactionsRequested()
+ .add(FateId.from(FateInstanceType.USER, UUID.randomUUID())));
+ }
+
+ @Test
+ public void testCompactionRequestedColumn() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+ FateInstanceType type = FateInstanceType.fromTableId(extent.tableId());
+ FateId userCompactFateId1 = FateId.from(type, UUID.randomUUID());
+ FateId userCompactFateId2 = FateId.from(type, UUID.randomUUID());
+
+ // Test column set
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId1.canonical(), "");
+ mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId2.canonical(), "");
+
+ TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(USER_COMPACTION_REQUESTED), true, false);
+ assertEquals(2, tm.getUserCompactionsRequested().size());
+ assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId1));
+ assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId2));
+
+ // Column not set
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(USER_COMPACTION_REQUESTED), true, false);
+ assertTrue(tm.getUserCompactionsRequested().isEmpty());
+
+ // Column not fetched
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(ColumnType.PREV_ROW), true, false);
+ assertThrows(IllegalStateException.class,
tm::getUserCompactionsRequested);
+ }
+
+ @Test
+ public void testUnsplittableColumn() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ StoredTabletFile sf1 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"));
+ StoredTabletFile sf2 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf2.rf"));
+ StoredTabletFile sf3 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf3.rf"));
+ // Same path as sf4 but with a range
- StoredTabletFile sf4 =
- StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf3.rf"), new Range("a", "b"));
++ StoredTabletFile sf4 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf3.rf"),
++ new Range("a", false, "b", true));
+
+ // Test with files
+ var unsplittableMeta1 =
+ UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1, sf2, sf3));
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta1.toBase64()));
+ TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta1, tm.getUnSplittable(), true);
+
+ // Test empty file set
+ var unsplittableMeta2 = UnSplittableMetadata.toUnSplittable(extent, 100,
110, 120, Set.of());
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta2.toBase64()));
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta2, tm.getUnSplittable(), true);
+
+ // Make sure not equals works as well
+ assertUnsplittable(unsplittableMeta1, unsplittableMeta2, false);
+
+ // Test with ranges
+ // use sf4 which includes sf4 instead of sf3 which has a range
+ var unsplittableMeta3 =
+ UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1, sf2, sf4));
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta3.toBase64()));
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertUnsplittable(unsplittableMeta3, tm.getUnSplittable(), true);
+
+ // make sure not equals when all the file paths are equal but one has a
range
+ assertUnsplittable(unsplittableMeta1, unsplittableMeta3, false);
+
+ // Column not set
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(UNSPLITTABLE), true, false);
+ assertNull(tm.getUnSplittable());
+ // Column not fetched
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+ EnumSet.of(ColumnType.PREV_ROW), true, false);
+ assertThrows(IllegalStateException.class, tm::getUnSplittable);
+ }
+
+ @Test
+ public void testUnsplittableWithRange() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ // Files with same path and different ranges
+ StoredTabletFile sf1 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"));
- StoredTabletFile sf2 =
- StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"), new Range("a", "b"));
- StoredTabletFile sf3 =
- StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"), new Range("a", "d"));
++ StoredTabletFile sf2 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"),
++ new Range("a", false, "b", true));
++ StoredTabletFile sf3 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"),
++ new Range("a", false, "d", true));
+
+ var meta1 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf1));
+ var meta2 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf2));
+ var meta3 = UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120,
Set.of(sf3));
+
+ // compare each against the others to make sure not equal
+ assertUnsplittable(meta1, meta2, false);
+ assertUnsplittable(meta1, meta3, false);
+ assertUnsplittable(meta2, meta3, false);
+ }
+
+ private void assertUnsplittable(UnSplittableMetadata meta1,
UnSplittableMetadata meta2,
+ boolean equal) {
+ assertEquals(equal, meta1.equals(meta2));
+ assertEquals(equal, meta1.hashCode() == meta2.hashCode());
+ assertEquals(equal, meta1.toBase64().equals(meta2.toBase64()));
+ }
+
+ @Test
+ public void testUnknownColFamily() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+
+ mutation.put("1234567890abcdefg", "xyz", "v1");
+ assertThrows(IllegalStateException.class, () -> TabletMetadata
+ .convertRow(toRowMap(mutation).entrySet().iterator(),
EnumSet.of(MERGED), true, false));
}
private SortedMap<Key,Value> toRowMap(Mutation mutation) {
diff --cc
server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
index b2aebb5b36,b8e7cf38a1..1424673095
---
a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
@@@ -367,12 -353,11 +367,11 @@@ public class MetadataConstraintsTest
// required for an endRow so will fail validation
m = new Mutation(new Text("0;foo"));
m.put(BulkFileColumnFamily.NAME,
- new Text(StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"), new Range("a", "b"))
- .getMetadata().replaceFirst("\"endRow\":\".*\"",
+ new Text(StoredTabletFile.of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"),
+ new Range("a", false, "b",
true)).getMetadata().replaceFirst("\"endRow\":\".*\"",
"\"endRow\":\"" + encodeRowForMetadata("bad") + "\"")),
- new Value("5"));
- assertViolation(mc, m, (short) 9);
+ new Value(fateId1.canonical()));
+ assertViolation(mc, m, (short) 12);
}
@@@ -453,12 -438,11 +452,11 @@@
// required for an endRow so this will fail validation
m = new Mutation(new Text("0;foo"));
m.put(columnFamily,
- new Text(StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"), new Range("a", "b"))
- .getMetadata()
+ new Text(StoredTabletFile.of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"),
+ new Range("a", false, "b", true)).getMetadata()
.replaceFirst("\"endRow\":\".*\"", "\"endRow\":\"" +
encodeRowForMetadata("b") + "\"")),
value);
- assertViolation(mc, m, (short) 9);
+ assertViolation(mc, m, (short) 12);
// Missing tables directory in path
m = new Mutation(new Text("0;foo"));
