This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 123ce967869621d527aff453bdf43f3617bc76ec
Merge: 31323e5e5e bc932655b3
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Tue Oct 10 17:51:13 2023 -0400
Merge remote-tracking branch 'upstream/main' into elasticity
.../org/apache/accumulo/core/conf/Property.java | 6 +-
.../accumulo/core/metadata/StoredTabletFile.java | 8 +
.../util/compaction/ExternalCompactionUtil.java | 16 +-
pom.xml | 49 ---
.../constraints/MetadataConstraintsTest.java | 210 +++++++---
server/compactor/pom.xml | 38 +-
.../apache/accumulo/compactor/CompactorTest.java | 455 ---------------------
.../src/test/resources/log4j2-test.properties | 35 --
server/native/pom.xml | 5 -
src/build/ci/find-unapproved-junit.sh | 8 +-
start/pom.xml | 17 +-
.../accumulo/test/functional/MonitorSslIT.java | 2 +-
.../accumulo/test/functional/ReadWriteIT.java | 2 +-
.../org/apache/accumulo/test/shell/MockShell.java | 17 +-
.../apache/accumulo/test/shell/ShellServerIT.java | 68 +--
15 files changed, 250 insertions(+), 686 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index e4eda5713d,cbda041d41..697f079d40
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@@ -99,15 -99,9 +99,9 @@@ public class ExternalCompactionUtil
* @return Optional HostAndPort of Coordinator node if found
*/
public static Optional<HostAndPort> findCompactionCoordinator(ClientContext
context) {
- final String lockPath = context.getZooKeeperRoot() +
Constants.ZCOORDINATOR_LOCK;
+ final String lockPath = context.getZooKeeperRoot() +
Constants.ZMANAGER_LOCK;
- try {
- var zk = ZooSession.getAnonymousSession(context.getZooKeepers(),
- context.getZooKeepersSessionTimeOut());
- return ServiceLock.getLockData(zk, ServiceLock.path(lockPath))
- .map(sld -> sld.getAddress(ThriftService.COORDINATOR));
- } catch (KeeperException | InterruptedException e) {
- throw new IllegalStateException(e);
- }
+ return ServiceLock.getLockData(context.getZooCache(),
ServiceLock.path(lockPath), new ZcStat())
+ .map(sld -> sld.getAddress(ThriftService.COORDINATOR));
}
/**
@@@ -115,14 -109,14 +109,14 @@@
*/
public static Map<String,List<HostAndPort>> getCompactorAddrs(ClientContext
context) {
try {
- final Map<String,List<HostAndPort>> queuesAndAddresses = new
HashMap<>();
- final String compactorQueuesPath = context.getZooKeeperRoot() +
Constants.ZCOMPACTORS;
+ final Map<String,List<HostAndPort>> groupsAndAddresses = new
HashMap<>();
+ final String compactorGroupsPath = context.getZooKeeperRoot() +
Constants.ZCOMPACTORS;
ZooReader zooReader = context.getZooReader();
- List<String> queues = zooReader.getChildren(compactorQueuesPath);
- for (String queue : queues) {
- queuesAndAddresses.putIfAbsent(queue, new ArrayList<>());
+ List<String> groups = zooReader.getChildren(compactorGroupsPath);
+ for (String group : groups) {
- groupsAndAddresses.putIfAbsent(group, new ArrayList<HostAndPort>());
++ groupsAndAddresses.putIfAbsent(group, new ArrayList<>());
try {
- List<String> compactors = zooReader.getChildren(compactorQueuesPath
+ "/" + queue);
+ List<String> compactors = zooReader.getChildren(compactorGroupsPath
+ "/" + group);
for (String compactor : compactors) {
// compactor is the address, we are checking to see if there is a
child node which
// represents the compactor's lock as a check that it's alive.
diff --cc pom.xml
index 7a5a7f1fde,602809f869..4fd8a15edd
--- a/pom.xml
+++ b/pom.xml
@@@ -322,12 -322,11 +321,6 @@@
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
-- <dependency>
- <!-- legacy junit version specified here for dependency convergence
-->
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.13.2</version>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-compaction-coordinator</artifactId>
- <version>${project.version}</version>
-- </dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-compactor</artifactId>
@@@ -623,37 -622,6 +616,11 @@@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <!-- converge transitive dependency version between powermock and
easymock -->
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- <version>3.3</version>
- </dependency>
+ <dependency>
+ <groupId>org.opentest4j</groupId>
+ <artifactId>opentest4j</artifactId>
+ <version>1.2.0</version>
+ </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-easymock</artifactId>
- <version>${version.powermock}</version>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-core</artifactId>
- <version>${version.powermock}</version>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>${version.powermock}</version>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-reflect</artifactId>
- <version>${version.powermock}</version>
- </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
diff --cc
server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
index f25361d8a9,f891a0b132..bbbccd2d72
---
a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
@@@ -22,10 -22,12 +22,13 @@@ import static org.junit.jupiter.api.Ass
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+ import java.lang.reflect.Method;
import java.net.URI;
+ import java.util.Base64;
import java.util.List;
+import java.util.Set;
+ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
@@@ -316,11 -337,7 +307,7 @@@ public class MetadataConstraintsTest
.of(URI.create("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"),
new Range())
.getMetadata().replace("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile",
"/someFile")),
new Value("5"));
- violations = mc.check(createEnv(), m);
- assertNotNull(violations);
- assertEquals(1, violations.size());
- assertEquals(Short.valueOf((short) 12), violations.get(0));
- assertNotNull(mc.getViolationDescription(violations.get(0)));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
// Missing tables directory in path
m = new Mutation(new Text("0;foo"));
@@@ -330,22 -347,76 +317,75 @@@
.getMetadata().replace("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile",
"hdfs://1.2.3.4/accumulo/2a/t-0003/someFile")),
new Value("5"));
- violations = mc.check(createEnv(), m);
- assertNotNull(violations);
- assertEquals(1, violations.size());
- assertEquals(Short.valueOf((short) 12), violations.get(0));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
- // No DataFileColumnFamily included
m = new Mutation(new Text("0;foo"));
m.put(BulkFileColumnFamily.NAME,
StoredTabletFile
.of(URI.create("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"), new
Range())
.getMetadataText(),
new Value("5"));
- violations = mc.check(createEnv(), m);
- assertNotNull(violations);
- assertEquals(1, violations.size());
- assertEquals(Short.valueOf((short) 8), violations.get(0));
- assertNotNull(mc.getViolationDescription(violations.get(0)));
+ assertViolation(mc, m, (short) 8);
+
+ // Bad Json - only path (old format) so should fail parsing
+ m = new Mutation(new Text("0;foo"));
+ m.put(BulkFileColumnFamily.NAME, new
Text("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"),
+ new Value("5"));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test startRow key is missing so validation should fail
+ //
{"path":"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(BulkFileColumnFamily.NAME,
+ new Text(
+
"{\"path\":\"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile\",\"endRow\":\"\"}"),
+ new Value("5"));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test path key replaced with empty string so validation
should fail
+ //
{"":"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile","startRow":"","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(
+ BulkFileColumnFamily.NAME, new Text(StoredTabletFile
+
.serialize("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile").replace("path",
"")),
+ new Value("5"));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test path value missing
+ // {"path":"","startRow":"","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(BulkFileColumnFamily.NAME,
+ new Text(StoredTabletFile
+
.of(URI.create("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"), new
Range())
+ .getMetadata().replaceFirst("\"path\":\".*\",\"startRow",
"\"path\":\"\",\"startRow")),
+ new Value("5"));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test startRow key replaced with empty string so validation
should fail
+ //
{"path":"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile","":"","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(BulkFileColumnFamily.NAME, new Text(StoredTabletFile
+
.serialize("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile").replace("startRow",
"")),
+ new Value("5"));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test endRow key missing so validation should fail
+ m = new Mutation(new Text("0;foo"));
+ m.put(
+ BulkFileColumnFamily.NAME, new Text(StoredTabletFile
+
.serialize("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile").replace("endRow",
"")),
+ new Value("5"));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - endRow will be replaced with encoded row without the
exclusive byte 0x00 which is
+ // required for an endRow so will fail validation
+ m = new Mutation(new Text("0;foo"));
+ m.put(BulkFileColumnFamily.NAME, new Text(StoredTabletFile
+ .of(URI.create("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"),
new Range("a", "b"))
+ .getMetadata()
+ .replaceFirst("\"endRow\":\".*\"", "\"endRow\":\"" +
encodeRowForMetadata("bad") + "\"")),
+ new Value("5"));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
}
@@@ -370,20 -441,67 +410,67 @@@
.of(URI.create("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"),
new Range())
.getMetadata().replace("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile",
"/someFile")),
value);
- violations = mc.check(createEnv(), m);
- assertNotNull(violations);
- assertEquals(1, violations.size());
- assertEquals(Short.valueOf((short) 12), violations.get(0));
- assertNotNull(mc.getViolationDescription(violations.get(0)));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
- // Bad Json - only path so should fail parsing
+ // Bad Json - only path (old format) so should fail parsing
m = new Mutation(new Text("0;foo"));
m.put(columnFamily, new
Text("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"), value);
- violations = mc.check(createEnv(), m);
- assertNotNull(violations);
- assertEquals(1, violations.size());
- assertEquals(Short.valueOf((short) 12), violations.get(0));
- assertNotNull(mc.getViolationDescription(violations.get(0)));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test path key replaced with empty string so validation
should fail
+ //
{"":"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile","startRow":"","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(
+ columnFamily, new Text(StoredTabletFile
+
.serialize("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile").replace("path",
"")),
+ value);
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test path value missing
+ // {"path":"","startRow":"","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(columnFamily,
+ new Text(StoredTabletFile
+
.of(URI.create("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"), new
Range())
+ .getMetadata().replaceFirst("\"path\":\".*\",\"startRow",
"\"path\":\"\",\"startRow")),
+ value);
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test startRow key replaced with empty string so validation
should fail
+ //
{"path":"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile","":"","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(columnFamily, new Text(StoredTabletFile
+
.serialize("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile").replace("startRow",
"")),
+ value);
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test startRow key is missing so validation should fail
+ //
{"path":"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(columnFamily,
+ new Text(
+
"{\"path\":\"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile\",\"endRow\":\"\"}"),
+ value);
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - test endRow key replaced with empty string so validation
should fail
+ //
{"path":"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile","":"","endRow":""}
+ m = new Mutation(new Text("0;foo"));
+ m.put(
+ columnFamily, new Text(StoredTabletFile
+
.serialize("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile").replace("endRow",
"")),
+ value);
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
+
+ // Bad Json - endRow will be replaced with encoded row without the
exclusive byte 0x00 which is
+ // required for an endRow so this will fail validation
+ m = new Mutation(new Text("0;foo"));
+ m.put(columnFamily, new Text(StoredTabletFile
+ .of(URI.create("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"),
new Range("a", "b"))
+ .getMetadata()
+ .replaceFirst("\"endRow\":\".*\"", "\"endRow\":\"" +
encodeRowForMetadata("b") + "\"")),
+ value);
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
// Missing tables directory in path
m = new Mutation(new Text("0;foo"));
@@@ -393,12 -511,9 +480,9 @@@
.getMetadata().replace("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile",
"hdfs://1.2.3.4/accumulo/2a/t-0003/someFile")),
new DataFileValue(1, 1).encodeAsValue());
- violations = mc.check(createEnv(), m);
- assertNotNull(violations);
- assertEquals(1, violations.size());
- assertEquals(Short.valueOf((short) 12), violations.get(0));
- assertViolation(mc, m, (short) 9);
++ assertViolation(mc, m, (short) 12);
- // Should pass validation
+ // Should pass validation (inf range)
m = new Mutation(new Text("0;foo"));
m.put(columnFamily,
StoredTabletFile
@@@ -408,66 -523,35 +492,92 @@@
violations = mc.check(createEnv(), m);
assertNull(violations);
+ // Should pass validation with range set
+ m = new Mutation(new Text("0;foo"));
+ m.put(columnFamily, StoredTabletFile
+ .of(URI.create("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile"),
new Range("a", "b"))
+ .getMetadataText(), new DataFileValue(1, 1).encodeAsValue());
+ violations = mc.check(createEnv(), m);
+ assertNull(violations);
+
- assertNotNull(mc.getViolationDescription((short) 9));
+ assertNotNull(mc.getViolationDescription((short) 12));
+ }
+
+ @Test
+ public void testOperationId() {
+ MetadataConstraints mc = new MetadataConstraints();
+ Mutation m;
+ List<Short> violations;
+
+ m = new Mutation(new Text("0;foo"));
+ ServerColumnFamily.OPID_COLUMN.put(m, new Value("bad id"));
- violations = mc.check(createEnv(), m);
- assertNotNull(violations);
- assertEquals(1, violations.size());
- assertEquals(Short.valueOf((short) 9), violations.get(0));
++ assertViolation(mc, m, (short) 9);
+
+ m = new Mutation(new Text("0;foo"));
+ ServerColumnFamily.OPID_COLUMN.put(m, new Value("MERGING:FATE[123abc]"));
+ violations = mc.check(createEnv(), m);
+ assertNull(violations);
+ }
+
+ @Test
+ public void testSelectedFiles() {
+ MetadataConstraints mc = new MetadataConstraints();
+ Mutation m;
+ List<Short> violations;
+
+ m = new Mutation(new Text("0;foo"));
+ ServerColumnFamily.SELECTED_COLUMN.put(m, new Value("bad id"));
+ violations = mc.check(createEnv(), m);
+ assertNotNull(violations);
+ assertEquals(1, violations.size());
+ assertEquals(Short.valueOf((short) 11), violations.get(0));
+
+ m = new Mutation(new Text("0;foo"));
+ ServerColumnFamily.SELECTED_COLUMN.put(m,
+ new Value(new SelectedFiles(Set.of(new ReferencedTabletFile(
+ new
Path("hdfs://nn.somewhere.com:86753/accumulo/tables/42/t-0000/F00001.rf"))
+ .insert()), true, 42L).getMetadataValue()));
+ violations = mc.check(createEnv(), m);
+ assertNull(violations);
+ }
+
+ @Test
+ public void testCompacted() {
+ MetadataConstraints mc = new MetadataConstraints();
+ Mutation m;
+ List<Short> violations;
+
+ m = new Mutation(new Text("0;foo"));
+ m.put(CompactedColumnFamily.STR_NAME, FateTxId.formatTid(45), "");
+ violations = mc.check(createEnv(), m);
+ assertNull(violations);
+
+ m = new Mutation(new Text("0;foo"));
+ m.put(CompactedColumnFamily.STR_NAME, "incorrect data", "");
+ violations = mc.check(createEnv(), m);
+ assertNotNull(violations);
+ assertEquals(1, violations.size());
+ assertEquals(Short.valueOf((short) 13), violations.get(0));
}
+
+ // Encode a row how it would appear in Json
+ private static String encodeRowForMetadata(String row) {
+ try {
+ Method method = StoredTabletFile.class.getDeclaredMethod("encodeRow",
Key.class);
+ method.setAccessible(true);
+ return Base64.getUrlEncoder()
+ .encodeToString((byte[]) method.invoke(StoredTabletFile.class, new
Key(row)));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void assertViolation(MetadataConstraints mc, Mutation m, Short
violation) {
+ List<Short> violations = mc.check(createEnv(), m);
+ assertNotNull(violations);
+ assertEquals(1, violations.size());
+ assertEquals(violation, violations.get(0));
+ assertNotNull(mc.getViolationDescription(violations.get(0)));
+ }
+
}
