This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push: new f9d79650fc Reduced default compaction configuration down to one group (#4299) f9d79650fc is described below commit f9d79650fcbc5bf430c573242e43de7e05b2dbe9 Author: Dave Marion <dlmar...@apache.org> AuthorDate: Tue Feb 27 09:26:09 2024 -0500 Reduced default compaction configuration down to one group (#4299) Fixes #3546 --- assemble/bin/accumulo-cluster | 6 +-- .../org/apache/accumulo/core/conf/Property.java | 7 ++- .../spi/compaction/DefaultCompactionPlanner.java | 14 +++--- .../compaction/DefaultCompactionPlannerTest.java | 53 +++++++++++----------- .../minicluster/MiniAccumuloClusterTest.java | 2 +- .../server/conf/CheckCompactionConfigTest.java | 38 ++++++++-------- .../compaction/BadCompactionServiceConfigIT.java | 8 ++-- .../test/compaction/CompactionConfigChangeIT.java | 6 +-- .../test/compaction/CompactionExecutorIT.java | 18 ++++---- .../CompactionPriorityQueueMetricsIT.java | 5 +- .../compaction/ExternalCompactionTestUtils.java | 16 +++---- .../accumulo/test/functional/CompactionIT.java | 6 +-- .../test/functional/FateConcurrencyIT.java | 5 +- .../accumulo/test/functional/FateStarvationIT.java | 9 ++-- .../test/functional/IdleProcessMetricsIT.java | 6 +-- .../test/functional/MemoryStarvedMajCIT.java | 11 +++-- 16 files changed, 104 insertions(+), 106 deletions(-) diff --git a/assemble/bin/accumulo-cluster b/assemble/bin/accumulo-cluster index 69879d2061..5a78c6773c 100755 --- a/assemble/bin/accumulo-cluster +++ b/assemble/bin/accumulo-cluster @@ -526,11 +526,7 @@ tserver: - localhost compactor: - accumulo_meta: - - localhost - user_small: - - localhost - user_large: + default: - localhost sserver: diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 394b79746e..f48253a812 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -65,7 +65,7 @@ public enum Property { COMPACTION_SERVICE_ROOT_MAX_OPEN(COMPACTION_SERVICE_PREFIX + "root.planner.opts.maxOpen", "30", PropertyType.COUNT, "The maximum number of files a compaction will open.", "4.0.0"), COMPACTION_SERVICE_ROOT_GROUPS(COMPACTION_SERVICE_PREFIX + "root.planner.opts.groups", - "[{'name':'accumulo_meta'}]".replaceAll("'", "\""), PropertyType.STRING, + "[{'group':'default'}]".replaceAll("'", "\""), PropertyType.JSON, "See {% jlink -f org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.", "4.0.0"), COMPACTION_SERVICE_META_PLANNER(COMPACTION_SERVICE_PREFIX + "meta.planner", @@ -74,7 +74,7 @@ public enum Property { COMPACTION_SERVICE_META_MAX_OPEN(COMPACTION_SERVICE_PREFIX + "meta.planner.opts.maxOpen", "30", PropertyType.COUNT, "The maximum number of files a compaction will open.", "4.0.0"), COMPACTION_SERVICE_META_GROUPS(COMPACTION_SERVICE_PREFIX + "meta.planner.opts.groups", - "[{'name':'accumulo_meta'}]".replaceAll("'", "\""), PropertyType.JSON, + "[{'group':'default'}]".replaceAll("'", "\""), PropertyType.JSON, "See {% jlink -f org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.", "4.0.0"), COMPACTION_SERVICE_DEFAULT_PLANNER(COMPACTION_SERVICE_PREFIX + "default.planner", @@ -83,8 +83,7 @@ public enum Property { COMPACTION_SERVICE_DEFAULT_MAX_OPEN(COMPACTION_SERVICE_PREFIX + "default.planner.opts.maxOpen", "10", PropertyType.COUNT, "The maximum number of files a compaction will open.", "4.0.0"), COMPACTION_SERVICE_DEFAULT_GROUPS(COMPACTION_SERVICE_PREFIX + "default.planner.opts.groups", - ("[{'name':'user_small','maxSize':'128M'}, {'name':'user_large'}]").replaceAll("'", "\""), - PropertyType.STRING, + ("[{'group':'default'}]").replaceAll("'", "\""), PropertyType.JSON, "See {% jlink -f org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.", "4.0.0"), COMPACTION_WARN_TIME(COMPACTION_PREFIX + "warn.time", "10m", PropertyType.TIMEDURATION, diff --git a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java index 2396ad85d3..0520db7304 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java @@ -72,8 +72,8 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; * <th>Description</th> * </tr> * <tr> - * <td>name</td> - * <td>name of the compactor group (required)</td> + * <td>group</td> + * <td>name of the compactor resource group (required)</td> * </tr> * <tr> * <td>maxSize</td> @@ -97,9 +97,9 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; * <pre> * {@code * [ - * {"name":"small", "maxSize":"100M"}, - * {"name":"medium", "maxSize":"500M"}, - * {"name": "large"} + * {"group":"small", "maxSize":"100M"}, + * {"group":"medium", "maxSize":"500M"}, + * {"group": "large"} * ]} * </pre> * </ul> @@ -127,7 +127,7 @@ public class DefaultCompactionPlanner implements CompactionPlanner { private final static Logger log = LoggerFactory.getLogger(DefaultCompactionPlanner.class); private static class GroupConfig { - String name; + String group; String maxSize; } @@ -192,7 +192,7 @@ public class DefaultCompactionPlanner implements CompactionPlanner { : ConfigurationTypeHelper.getFixedMemoryAsBytes(groupConfig.maxSize); CompactorGroupId cgid; - String group = Objects.requireNonNull(groupConfig.name, "'name' must be specified"); + String group = Objects.requireNonNull(groupConfig.group, "'group' must be specified"); cgid = params.getGroupManager().getGroup(group); tmpGroups.add(new CompactionGroup(cgid, maxSize)); } diff --git a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java index d0f3e77eac..43a6179b51 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java @@ -152,8 +152,8 @@ public class DefaultCompactionPlannerTest { @Test public void testRunningCompaction() { - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large','maxSize':'512M'}, {'name':'huge'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large','maxSize':'512M'}, {'group':'huge'}]"; var planner = createPlanner(defaultConf, groups); @@ -187,8 +187,8 @@ public class DefaultCompactionPlannerTest { aconf.set(prefix + "cs1.planner.opts.maxOpen", "15"); ConfigurationImpl config = new ConfigurationImpl(aconf); - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large','maxSize':'512M'}, {'name':'huge'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large','maxSize':'512M'}, {'group':'huge'}]"; var planner = createPlanner(config, groups); var all = createCFs("F1", "3M", "F2", "3M", "F3", "11M", "F4", "12M", "F5", "13M"); @@ -256,8 +256,8 @@ public class DefaultCompactionPlannerTest { @Test public void testMaxSize() { - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large','maxSize':'512M'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large','maxSize':'512M'}]"; var planner = createPlanner(defaultConf, groups); var all = createCFs("F1", "128M", "F2", "129M", "F3", "130M", "F4", "131M", "F5", "132M"); @@ -281,8 +281,8 @@ public class DefaultCompactionPlannerTest { public void testMultipleCompactions() { // This test validates that when a tablet has many files that multiple compaction jobs can be // issued at the same time. - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large','maxSize':'512M'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large','maxSize':'512M'}]"; for (var kind : List.of(CompactionKind.USER, CompactionKind.SYSTEM)) { var planner = createPlanner(defaultConf, groups); @@ -315,8 +315,8 @@ public class DefaultCompactionPlannerTest { @Test public void testMultipleCompactionsAndLargeCompactionRatio() { - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large','maxSize':'512M'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large','maxSize':'512M'}]"; var planner = createPlanner(defaultConf, groups); var all = IntStream.range(0, 65).mapToObj(i -> createCF("F" + i, i + 1)).collect(toSet()); // This compaction ratio would not cause a system compaction, how a user compaction must compact @@ -349,8 +349,8 @@ public class DefaultCompactionPlannerTest { // This test validates that when a tablet has many files that multiple compaction jobs can be // issued at the same time even if there are running compaction as long everything meets the // compaction ratio. - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large','maxSize':'512M'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large','maxSize':'512M'}]"; for (var kind : List.of(CompactionKind.USER, CompactionKind.SYSTEM)) { var planner = createPlanner(defaultConf, groups); var all = IntStream.range(0, 990).mapToObj(i -> createCF("F" + i, 1000)).collect(toSet()); @@ -391,8 +391,8 @@ public class DefaultCompactionPlannerTest { public void testUserCompactionDoesNotWaitOnSystemCompaction() { // this test ensures user compactions do not wait on system compactions to complete - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large','maxSize':'512M'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large','maxSize':'512M'}]"; var planner = createPlanner(defaultConf, groups); var all = createCFs("F1", "1M", "F2", "1M", "F3", "1M", "F4", "3M", "F5", "3M", "F6", "3M", "F7", "20M"); @@ -435,7 +435,7 @@ public class DefaultCompactionPlannerTest { public void testQueueCreation() { DefaultCompactionPlanner planner = new DefaultCompactionPlanner(); - String groups = "[{\"name\": \"small\", \"maxSize\":\"32M\"},{\"name\":\"midsize\"}]"; + String groups = "[{\"group\": \"small\", \"maxSize\":\"32M\"},{\"group\":\"midsize\"}]"; planner.init(getInitParams(defaultConf, groups)); var all = createCFs("F1", "1M", "F2", "1M", "F3", "1M", "F4", "1M"); @@ -463,7 +463,7 @@ public class DefaultCompactionPlannerTest { DefaultCompactionPlanner planner = new DefaultCompactionPlanner(); String groups = - "[{\"name\":\"smallQueue\", \"maxSize\":\"32M\"}, {\"name\":\"largeQueue\", \"type\":\"internal\", \"foo\":\"bar\", \"queue\":\"broken\"}]"; + "[{\"group\":\"smallQueue\", \"maxSize\":\"32M\"}, {\"group\":\"largeQueue\", \"type\":\"internal\", \"foo\":\"bar\", \"queue\":\"broken\"}]"; final InitParameters params = getInitParams(defaultConf, groups); assertNotNull(params); @@ -480,14 +480,15 @@ public class DefaultCompactionPlannerTest { @Test public void testErrorGroupNoName() { DefaultCompactionPlanner planner = new DefaultCompactionPlanner(); - String groups = "[{\"name\":\"smallQueue\", \"maxSize\":\"32M\"}, {\"maxSize\":\"120M\"}]"; + String groups = "[{\"group\":\"smallQueue\", \"maxSize\":\"32M\"}, {\"maxSize\":\"120M\"}]"; final InitParameters params = getInitParams(defaultConf, groups); assertNotNull(params); var e = assertThrows(NullPointerException.class, () -> planner.init(params), "Failed to throw error"); - assertEquals(e.getMessage(), "'name' must be specified", "Error message didn't contain 'name'"); + assertEquals(e.getMessage(), "'group' must be specified", + "Error message didn't contain 'group'"); } /** @@ -512,7 +513,7 @@ public class DefaultCompactionPlannerTest { public void testErrorOnlyOneMaxSize() { DefaultCompactionPlanner planner = new DefaultCompactionPlanner(); String groups = - "[{\"name\":\"small\", \"maxSize\":\"32M\"}, {\"name\":\"medium\"}, {\"name\":\"large\"}]"; + "[{\"group\":\"small\", \"maxSize\":\"32M\"}, {\"group\":\"medium\"}, {\"group\":\"large\"}]"; var e = assertThrows(IllegalArgumentException.class, () -> planner.init(getInitParams(defaultConf, groups)), "Failed to throw error"); assertTrue(e.getMessage().contains("Can only have one group w/o a maxSize"), @@ -526,7 +527,7 @@ public class DefaultCompactionPlannerTest { public void testErrorDuplicateMaxSize() { DefaultCompactionPlanner planner = new DefaultCompactionPlanner(); String groups = - "[{\"name\":\"small\", \"maxSize\":\"32M\"}, {\"name\":\"medium\", \"maxSize\":\"32M\"}, {\"name\":\"large\"}]"; + "[{\"group\":\"small\", \"maxSize\":\"32M\"}, {\"group\":\"medium\", \"maxSize\":\"32M\"}, {\"group\":\"large\"}]"; var e = assertThrows(IllegalArgumentException.class, () -> planner.init(getInitParams(defaultConf, groups)), "Failed to throw error"); assertTrue(e.getMessage().contains("Duplicate maxSize set in groups"), @@ -538,8 +539,8 @@ public class DefaultCompactionPlannerTest { // compaction. @Test public void testMaxTabletFiles() { - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large'}]"; Map<String,String> overrides = new HashMap<>(); overrides.put(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner.opts.maxOpen", "10"); @@ -614,8 +615,8 @@ public class DefaultCompactionPlannerTest { @Test public void testMaxTabletFilesNoCompaction() { - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large', 'maxSize':'512M'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large', 'maxSize':'512M'}]"; Map<String,String> overrides = new HashMap<>(); overrides.put(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner.opts.maxOpen", "10"); @@ -651,8 +652,8 @@ public class DefaultCompactionPlannerTest { // Test to ensure that plugin falls back from TABLE_FILE_MAX to TSERV_SCAN_MAX_OPENFILES @Test public void testMaxTableFilesFallback() { - String groups = "[{'name':'small','maxSize':'32M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large'}]"; + String groups = "[{'group':'small','maxSize':'32M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large'}]"; Map<String,String> overrides = new HashMap<>(); overrides.put(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner.opts.maxOpen", "10"); diff --git a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java index 3d9621ad90..cd19670dfb 100644 --- a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java +++ b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java @@ -187,7 +187,7 @@ public class MiniAccumuloClusterTest extends WithTestNames { public void testDebugPorts() { Set<Pair<ServerType,Integer>> debugPorts = accumulo.getDebugPorts(); - assertEquals(10, debugPorts.size()); + assertEquals(7, debugPorts.size()); for (Pair<ServerType,Integer> debugPort : debugPorts) { assertTrue(debugPort.getSecond() > 0); } diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java index 052bb9c6f8..5490a21ee1 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java @@ -52,8 +52,8 @@ public class CheckCompactionConfigTest extends WithTestNames { String inputString = ("compaction.service.cs1.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs1.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'},{'name':'medium','maxSize':'128M'},\\\n" - + "{'name':'large'}]").replaceAll("'", "\""); + + "[{'group':'small','maxSize':'16M'},{'group':'medium','maxSize':'128M'},\\\n" + + "{'group':'large'}]").replaceAll("'", "\""); String filePath = writeToFileAndReturnPath(inputString); @@ -65,12 +65,12 @@ public class CheckCompactionConfigTest extends WithTestNames { String inputString = ("compaction.service.cs1.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs1.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'},{'name':'medium','maxSize':'128M'},\\\n" - + "{'name':'large'}] \ncompaction.service.cs2.planner=" + + "[{'group':'small','maxSize':'16M'},{'group':'medium','maxSize':'128M'},\\\n" + + "{'group':'large'}] \ncompaction.service.cs2.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs2.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'},{'name':'medium','maxSize':'128M'},\\\n" - + "{'name':'large'}]").replaceAll("'", "\""); + + "[{'group':'small','maxSize':'16M'},{'group':'medium','maxSize':'128M'},\\\n" + + "{'group':'large'}]").replaceAll("'", "\""); String filePath = writeToFileAndReturnPath(inputString); @@ -82,15 +82,15 @@ public class CheckCompactionConfigTest extends WithTestNames { String inputString = ("compaction.service.cs1.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs1.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'},{'name':'medium','maxSize':'128M'},\\\n" - + "{'name':'large'}] \ncompaction.service.cs2.planner=" + + "[{'group':'small','maxSize':'16M'},{'group':'medium','maxSize':'128M'},\\\n" + + "{'group':'large'}] \ncompaction.service.cs2.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs2.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'}, {'name':'medium','maxSize':'128M'},\\\n" - + "{'name':'large'}] \ncompaction.service.cs3.planner=" + + "[{'group':'small','maxSize':'16M'}, {'group':'medium','maxSize':'128M'},\\\n" + + "{'group':'large'}] \ncompaction.service.cs3.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs3.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'},{'name':'large'}]").replaceAll("'", "\""); + + "[{'group':'small','maxSize':'16M'},{'group':'large'}]").replaceAll("'", "\""); String filePath = writeToFileAndReturnPath(inputString); CheckCompactionConfig.main(new String[] {filePath}); @@ -101,8 +101,8 @@ public class CheckCompactionConfigTest extends WithTestNames { String inputString = ("compaction.service.cs1.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs1.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'},{'name':'medium','maxSize':'128M'},\\\n" - + "{'name':'large','numThreads':2}]").replaceAll("'", "\""); + + "[{'group':'small','maxSize':'16M'},{'group':'medium','maxSize':'128M'},\\\n" + + "{'group':'large','numThreads':2}]").replaceAll("'", "\""); String expectedErrorMsg = "Invalid fields: [numThreads] provided for class: org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner$GroupConfig"; @@ -116,8 +116,8 @@ public class CheckCompactionConfigTest extends WithTestNames { @Test public void testNoPlanner() throws Exception { String inputString = ("compaction.service.cs1.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'}, {'name':'medium','maxSize':'128M'},\\\n" - + "{'name':'large'}]").replaceAll("'", "\""); + + "[{'group':'small','maxSize':'16M'}, {'group':'medium','maxSize':'128M'},\\\n" + + "{'group':'large'}]").replaceAll("'", "\""); String expectedErrorMsg = "Incomplete compaction service definitions, missing planner class [cs1]"; @@ -133,8 +133,8 @@ public class CheckCompactionConfigTest extends WithTestNames { String inputString = ("compaction.service.cs1.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs1.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'},{'name':'medium','maxSize':'128M'},\\\n" - + "{'name':'small'}]").replaceAll("'", "\""); + + "[{'group':'small','maxSize':'16M'},{'group':'medium','maxSize':'128M'},\\\n" + + "{'group':'small'}]").replaceAll("'", "\""); String expectedErrorMsg = "Duplicate compactor group for group: small"; final String filePath = writeToFileAndReturnPath(inputString); @@ -149,8 +149,8 @@ public class CheckCompactionConfigTest extends WithTestNames { String inputString = ("compaction.service.cs1.planner=" + "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n" + "compaction.service.cs1.planner.opts.groups=\\\n" - + "[{'name':'small','maxSize':'16M'},{'name':'medium','maxSize':'0M'},\\\n" - + "{'name':'large'}]").replaceAll("'", "\""); + + "[{'group':'small','maxSize':'16M'},{'group':'medium','maxSize':'0M'},\\\n" + + "{'group':'large'}]").replaceAll("'", "\""); String expectedErrorMsg = "Invalid value for maxSize"; String filePath = writeToFileAndReturnPath(inputString); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java index 4bdd5a8eca..1aaa507fb3 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java @@ -70,10 +70,10 @@ public class BadCompactionServiceConfigIT extends SharedMiniClusterBase { public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { Map<String,String> siteCfg = new HashMap<>(); siteCfg.put(CSP + "default.planner", DefaultCompactionPlanner.class.getName()); - siteCfg.put(CSP + "default.planner.opts.groups", "[{\"name\":\"default_group\"}]"); + siteCfg.put(CSP + "default.planner.opts.groups", "[{\"group\":\"default_group\"}]"); siteCfg.put(CSP + "cs1.planner", DefaultCompactionPlanner.class.getName()); // place invalid json in the planners config - siteCfg.put(CSP + "cs1.planner.opts.groups", "{{'name]"); + siteCfg.put(CSP + "cs1.planner.opts.groups", "{{'group]"); cfg.setSiteConfig(siteCfg); } } @@ -128,7 +128,7 @@ public class BadCompactionServiceConfigIT extends SharedMiniClusterBase { .collect(MoreCollectors.onlyElement())); } - var value = "[{'name':'cs1q1'}]".replaceAll("'", "\""); + var value = "[{'group':'cs1q1'}]".replaceAll("'", "\""); client.instanceOperations().setProperty(CSP + "cs1.planner.opts.groups", value); // start the compactor, it was not started initially because of bad config @@ -166,7 +166,7 @@ public class BadCompactionServiceConfigIT extends SharedMiniClusterBase { fixerFuture = executorService.submit(() -> { try { Thread.sleep(2000); - var value = "[{'name':'cs1q1'}]".replaceAll("'", "\""); + var value = "[{'group':'cs1q1'}]".replaceAll("'", "\""); client.instanceOperations().setProperty(CSP + "cs1.planner.opts.groups", value); } catch (Exception e) { throw new RuntimeException(e); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java index 273d6c93b1..d0e5797505 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java @@ -51,8 +51,8 @@ public class CompactionConfigChangeIT extends AccumuloClusterHarness { cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner.opts.groups", - ("[{'name':'small','maxSize':'2M'}, {'name':'medium','maxSize':'128M'}," - + "{'name':'large'}]").replaceAll("'", "\"")); + ("[{'group':'small','maxSize':'2M'}, {'group':'medium','maxSize':'128M'}," + + "{'group':'large'}]").replaceAll("'", "\"")); // use raw local file system so walogs sync and flush will work hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); @@ -102,7 +102,7 @@ public class CompactionConfigChangeIT extends AccumuloClusterHarness { // with running compactions. client.instanceOperations().setProperty( Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner.opts.groups", - ("[{'name':'little','maxSize':'128M'},{'name':'big'}]").replaceAll("'", "\"")); + ("[{'group':'little','maxSize':'128M'},{'group':'big'}]").replaceAll("'", "\"")); Wait.waitFor(() -> countFiles(client, table, "F") == 0, 60000); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java index 10d4dd1cd8..67c01286a9 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java @@ -94,7 +94,7 @@ public class CompactionExecutorIT extends SharedMiniClusterBase { public static class TestPlanner implements CompactionPlanner { private static class GroupConfig { - String name; + String group; } private int filesPerCompaction; @@ -112,7 +112,7 @@ public class CompactionExecutorIT extends SharedMiniClusterBase { for (JsonElement element : GSON.get().fromJson(groups, JsonArray.class)) { GroupConfig groupConfig = GSON.get().fromJson(element, GroupConfig.class); - var cgid = params.getGroupManager().getGroup(groupConfig.name); + var cgid = params.getGroupManager().getGroup(groupConfig.group); groupIds.add(cgid); } @@ -158,28 +158,28 @@ public class CompactionExecutorIT extends SharedMiniClusterBase { var csp = Property.COMPACTION_SERVICE_PREFIX.getKey(); cfg.setProperty(csp + "cs1.planner", TestPlanner.class.getName()); cfg.setProperty(csp + "cs1.planner.opts.groups", - "[{'name':'e1'},{'name':'e2'},{'name':'e3'}]"); + "[{'group':'e1'},{'group':'e2'},{'group':'e3'}]"); cfg.setProperty(csp + "cs1.planner.opts.filesPerCompaction", "5"); cfg.setProperty(csp + "cs1.planner.opts.process", "SYSTEM"); cfg.setProperty(csp + "cs2.planner", TestPlanner.class.getName()); - cfg.setProperty(csp + "cs2.planner.opts.groups", "[{'name':'f1'},{'name':'f2'}]"); + cfg.setProperty(csp + "cs2.planner.opts.groups", "[{'group':'f1'},{'group':'f2'}]"); cfg.setProperty(csp + "cs2.planner.opts.filesPerCompaction", "7"); cfg.setProperty(csp + "cs2.planner.opts.process", "SYSTEM"); cfg.setProperty(csp + "cs3.planner", TestPlanner.class.getName()); - cfg.setProperty(csp + "cs3.planner.opts.groups", "[{'name':'g1'}]"); + cfg.setProperty(csp + "cs3.planner.opts.groups", "[{'group':'g1'}]"); cfg.setProperty(csp + "cs3.planner.opts.filesPerCompaction", "3"); cfg.setProperty(csp + "cs3.planner.opts.process", "USER"); cfg.setProperty(csp + "cs4.planner", TestPlanner.class.getName()); - cfg.setProperty(csp + "cs4.planner.opts.groups", "[{'name':'h1'},{'name':'h2'}]"); + cfg.setProperty(csp + "cs4.planner.opts.groups", "[{'group':'h1'},{'group':'h2'}]"); cfg.setProperty(csp + "cs4.planner.opts.filesPerCompaction", "11"); cfg.setProperty(csp + "cs4.planner.opts.process", "USER"); // this is meant to be dynamically reconfigured cfg.setProperty(csp + "recfg.planner", TestPlanner.class.getName()); - cfg.setProperty(csp + "recfg.planner.opts.groups", "[{'name':'i1'},{'name':'i2'}]"); + cfg.setProperty(csp + "recfg.planner.opts.groups", "[{'group':'i1'},{'group':'i2'}]"); cfg.setProperty(csp + "recfg.planner.opts.filesPerCompaction", "11"); cfg.setProperty(csp + "recfg.planner.opts.process", "SYSTEM"); @@ -254,7 +254,7 @@ public class CompactionExecutorIT extends SharedMiniClusterBase { "5"); client.instanceOperations().setProperty( Property.COMPACTION_SERVICE_PREFIX.getKey() + "recfg.planner.opts.groups", - "[{'name':'group1'}]"); + "[{'group':'group1'}]"); getCluster().getConfig().getClusterServerConfiguration().addCompactorResourceGroup("group1", 1); compactionGroups.add("group1"); @@ -280,7 +280,7 @@ public class CompactionExecutorIT extends SharedMiniClusterBase { Property.COMPACTION_SERVICE_PREFIX.getKey() + "newcs.planner.opts.process", "SYSTEM"); client.instanceOperations().setProperty( Property.COMPACTION_SERVICE_PREFIX.getKey() + "newcs.planner.opts.groups", - "[{'name':'add1'},{'name':'add2'}, {'name':'add3'}]"); + "[{'group':'add1'},{'group':'add2'}, {'group':'add3'}]"); client.instanceOperations().setProperty( Property.COMPACTION_SERVICE_PREFIX.getKey() + "newcs.planner", TestPlanner.class.getName()); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java index a1e45815f0..d202df8b16 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java @@ -151,15 +151,14 @@ public class CompactionPriorityQueueMetricsIT extends SharedMiniClusterBase { @Override public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration conf) { cfg.setMemory(ServerType.TABLET_SERVER, 512, MemoryUnit.MEGABYTE); - // Zero the default compactors - cfg.getClusterServerConfiguration().setNumDefaultCompactors(0); + cfg.getClusterServerConfiguration().setNumDefaultCompactors(1); // Create a new queue with zero compactors. cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + QUEUE1_SERVICE + ".planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty( Property.COMPACTION_SERVICE_PREFIX.getKey() + QUEUE1_SERVICE + ".planner.opts.groups", - "[{'name':'" + QUEUE1 + "'}]"); + "[{'group':'" + QUEUE1 + "'}]"); cfg.setProperty(Property.MANAGER_COMPACTION_SERVICE_PRIORITY_QUEUE_SIZE, "6"); cfg.getClusterServerConfiguration().addCompactorResourceGroup(QUEUE1, 0); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java index d36b70fad5..c9cd441c18 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java @@ -198,35 +198,35 @@ public class ExternalCompactionTestUtils { cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner.opts.groups", - "[{'name':'" + GROUP1 + "'}]"); + "[{'group':'" + GROUP1 + "'}]"); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs2.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs2.planner.opts.groups", - "[{'name':'" + GROUP2 + "'}]"); + "[{'group':'" + GROUP2 + "'}]"); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs3.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs3.planner.opts.groups", - "[{'name':'" + GROUP3 + "'}]"); + "[{'group':'" + GROUP3 + "'}]"); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs4.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs4.planner.opts.groups", - "[{'name':'" + GROUP4 + "'}]"); + "[{'group':'" + GROUP4 + "'}]"); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs5.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs5.planner.opts.groups", - "[{'name':'" + GROUP5 + "'}]"); + "[{'group':'" + GROUP5 + "'}]"); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs6.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs6.planner.opts.groups", - "[{'name':'" + GROUP6 + "'}]"); + "[{'group':'" + GROUP6 + "'}]"); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs7.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs7.planner.opts.groups", - "[{'name':'" + GROUP7 + "'}]"); + "[{'group':'" + GROUP7 + "'}]"); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs8.planner", DefaultCompactionPlanner.class.getName()); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs8.planner.opts.groups", - "[{'name':'" + GROUP8 + "'}]"); + "[{'group':'" + GROUP8 + "'}]"); cfg.setProperty(Property.COMPACTION_COORDINATOR_FINALIZER_COMPLETION_CHECK_INTERVAL, "5s"); cfg.setProperty(Property.COMPACTION_COORDINATOR_DEAD_COMPACTOR_CHECK_INTERVAL, "5s"); cfg.setProperty(Property.COMPACTION_COORDINATOR_TSERVER_COMPACTION_CHECK_INTERVAL, "3s"); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java index 4a06574cfc..5ba42775f1 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java @@ -839,7 +839,7 @@ public class CompactionIT extends AccumuloClusterHarness { DefaultCompactionPlanner.class.getName()); c.instanceOperations().setProperty( Property.COMPACTION_SERVICE_PREFIX.getKey() + "deleteme.planner.opts.groups", - ("[{'name':'" + COMPACTOR_GROUP_1 + "'}]").replaceAll("'", "\"")); + ("[{'group':'" + COMPACTOR_GROUP_1 + "'}]").replaceAll("'", "\"")); // create a compaction service named keepme c.instanceOperations().setProperty( @@ -847,7 +847,7 @@ public class CompactionIT extends AccumuloClusterHarness { DefaultCompactionPlanner.class.getName()); c.instanceOperations().setProperty( Property.COMPACTION_SERVICE_PREFIX.getKey() + "keepme.planner.opts.groups", - ("[{'name':'" + COMPACTOR_GROUP_2 + "'}]").replaceAll("'", "\"")); + ("[{'group':'" + COMPACTOR_GROUP_2 + "'}]").replaceAll("'", "\"")); // create a table that uses the compaction service deleteme Map<String,String> props = new HashMap<>(); @@ -887,7 +887,7 @@ public class CompactionIT extends AccumuloClusterHarness { DefaultCompactionPlanner.class.getName()); c.instanceOperations().setProperty( Property.COMPACTION_SERVICE_PREFIX.getKey() + "newcs.planner.opts.groups", - ("[{'name':'" + COMPACTOR_GROUP_2 + "'}]").replaceAll("'", "\"")); + ("[{'group':'" + COMPACTOR_GROUP_2 + "'}]").replaceAll("'", "\"")); // set table 1 to a compaction service newcs c.tableOperations().setProperty(table1, diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java b/test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java index 68c24ff3cb..325f905129 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java @@ -516,9 +516,10 @@ public class FateConcurrencyIT extends AccumuloClusterHarness { int tableCount = 4; - // Start 4 Compactors for the user_small group + // Start 4 Compactors for the default group MiniAccumuloClusterImpl mini = (MiniAccumuloClusterImpl) getCluster(); - mini.getConfig().getClusterServerConfiguration().addCompactorResourceGroup("user_small", 4); + mini.getConfig().getClusterServerConfiguration() + .addCompactorResourceGroup(Constants.DEFAULT_RESOURCE_GROUP_NAME, 4); mini.start(); List<SlowOps> tables = Arrays.stream(getUniqueNames(tableCount)) diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java b/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java index 6b6da95020..69afdfe943 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.admin.NewTableConfiguration; @@ -53,13 +54,13 @@ public class FateStarvationIT extends AccumuloClusterHarness { @Override public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { - var groupName = "user_small"; // Add this check in case the config changes - Preconditions.checkState( - Property.COMPACTION_SERVICE_DEFAULT_GROUPS.getDefaultValue().contains(groupName)); + Preconditions.checkState(Property.COMPACTION_SERVICE_DEFAULT_GROUPS.getDefaultValue() + .contains(Constants.DEFAULT_RESOURCE_GROUP_NAME)); // This test creates around ~1300 compaction task, so start more compactors. There is randomness // so the exact number of task varies. - cfg.getClusterServerConfiguration().addCompactorResourceGroup(groupName, 4); + cfg.getClusterServerConfiguration() + .addCompactorResourceGroup(Constants.DEFAULT_RESOURCE_GROUP_NAME, 4); } @Test diff --git a/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java b/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java index 0f214f218c..169d54a3c2 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java @@ -45,11 +45,11 @@ public class IdleProcessMetricsIT extends SharedMiniClusterBase { // Configure all compaction planners to use the default resource group so // that only 1 compactor is started by MiniAccumuloCluster cfg.setProperty("compaction.service.root.planner.opts.groups", - "[{'name':'default'}]".replaceAll("'", "\"")); + "[{'group':'default'}]".replaceAll("'", "\"")); cfg.setProperty("compaction.service.meta.planner.opts.groups", - "[{'name':'default'}]".replaceAll("'", "\"")); + "[{'group':'default'}]".replaceAll("'", "\"")); cfg.setProperty("compaction.service.default.planner.opts.groups", - "[{'name':'default'}]".replaceAll("'", "\"")); + "[{'group':'default'}]".replaceAll("'", "\"")); // Disable the default scan servers and compactors, just start 1 // tablet server in the default group to host the root and metadata diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java index 19723a5391..03f7442c91 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.DoubleAdder; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.admin.CompactionConfig; @@ -153,14 +154,14 @@ public class MemoryStarvedMajCIT extends SharedMiniClusterBase { // Start the Compactors that will consume and free memory when we need it to getCluster().getClusterControl().start(ServerType.COMPACTOR, null, 1, MemoryConsumingCompactor.class); - Wait.waitFor(() -> ExternalCompactionUtil.getCompactorAddrs(ctx).size() == 4, 60_000); - Wait.waitFor( - () -> ExternalCompactionUtil.getCompactorAddrs(ctx).get("user_small").size() == 1, - 60_000); + Wait.waitFor(() -> ExternalCompactionUtil.getCompactorAddrs(ctx).size() == 1, 60_000); + Wait.waitFor(() -> ExternalCompactionUtil.getCompactorAddrs(ctx) + .get(Constants.DEFAULT_RESOURCE_GROUP_NAME).size() == 1, 60_000); Map<String,List<HostAndPort>> groupedCompactors = ExternalCompactionUtil.getCompactorAddrs(ctx); - List<HostAndPort> compactorAddresses = groupedCompactors.get("user_small"); + List<HostAndPort> compactorAddresses = + groupedCompactors.get(Constants.DEFAULT_RESOURCE_GROUP_NAME); HostAndPort compactorAddr = compactorAddresses.get(0); TableOperations to = client.tableOperations();