This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a3b346c02462ef948f6b5ac20aa4a434eca6d4aa
Merge: bad0f89ca8 87ab1f73b0
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Wed Oct 9 22:44:33 2024 +0000
Merge branch '3.1'
.../accumulo/core/client/PluginEnvironment.java | 17 +++++++++++++++
.../core/spi/common/ServiceEnvironment.java | 12 +++++++++++
.../balancer/HostRegexTableLoadBalancerTest.java | 8 ++------
.../core/spi/balancer/TableLoadBalancerTest.java | 24 +++++++++-------------
.../RatioBasedCompactionPlannerTest.java | 17 ++++++---------
5 files changed, 47 insertions(+), 31 deletions(-)
diff --cc
core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java
index f3927dbc3f,5071ddea12..6507c91566
---
a/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java
@@@ -39,9 -39,6 +39,7 @@@ import java.util.Set
import java.util.SortedMap;
import java.util.regex.Pattern;
+import org.apache.accumulo.core.Constants;
- import org.apache.accumulo.core.conf.ConfigurationCopy;
- import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.TabletId;
import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
diff --cc
core/src/test/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancerTest.java
index 0ce1cff351,06a30d88ed..8974924e53
---
a/core/src/test/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancerTest.java
@@@ -36,8 -34,6 +36,7 @@@ import java.util.SortedMap
import java.util.TreeMap;
import java.util.stream.Collectors;
+import org.apache.accumulo.core.Constants;
- import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.TabletId;
@@@ -172,96 -165,4 +171,93 @@@ public class TableLoadBalancerTest
}
}
+ private static class TestCurrAssignment implements
TabletBalancer.CurrentAssignment {
+
+ private final TabletIdImpl tablet;
+ private final String resourceGroup;
+
+ TestCurrAssignment(TableId tid, String rg) {
+ this.tablet = new TabletIdImpl(new KeyExtent(tid, null, null));
+ this.resourceGroup = rg;
+ }
+
+ @Override
+ public TabletId getTablet() {
+ return tablet;
+ }
+
+ @Override
+ public TabletServerId getTabletServer() {
+ return null;
+ }
+
+ @Override
+ public String getResourceGroup() {
+ return resourceGroup;
+ }
+ }
+
+ @Test
+ public void testNeedsReassignment() {
+
- ConfigurationCopy cc1 =
- new
ConfigurationCopy(Map.of(TableLoadBalancer.TABLE_ASSIGNMENT_GROUP_PROPERTY,
"G1"));
- ConfigurationImpl table1Config = new ConfigurationImpl(cc1);
-
- ConfigurationCopy cc2 =
- new
ConfigurationCopy(Map.of(TableLoadBalancer.TABLE_ASSIGNMENT_GROUP_PROPERTY,
"G2"));
- ConfigurationImpl table2Config = new ConfigurationImpl(cc2);
++ var table1Config = ServiceEnvironment.Configuration
++ .from(Map.of(TableLoadBalancer.TABLE_ASSIGNMENT_GROUP_PROPERTY,
"G1"), false);
++ var table2Config = ServiceEnvironment.Configuration
++ .from(Map.of(TableLoadBalancer.TABLE_ASSIGNMENT_GROUP_PROPERTY,
"G2"), false);
++ var table3Config = ServiceEnvironment.Configuration.from(Map.of(), false);
+
+ var tid1 = TableId.of("1");
+ var tid2 = TableId.of("2");
+ var tid3 = TableId.of("3");
+
+ BalancerEnvironment environment = createMock(BalancerEnvironment.class);
+
expect(environment.getConfiguration(tid1)).andReturn(table1Config).anyTimes();
+
expect(environment.getConfiguration(tid2)).andReturn(table2Config).anyTimes();
- expect(environment.getConfiguration(tid3))
- .andReturn(new ConfigurationImpl(new ConfigurationCopy())).anyTimes();
++
expect(environment.getConfiguration(tid3)).andReturn(table3Config).anyTimes();
+ replay(environment);
+
+ var tls = new TableLoadBalancer() {
+ @Override
+ protected TabletBalancer getBalancerForTable(TableId tableId) {
+ TabletBalancer balancer = createMock(TabletBalancer.class);
+ expect(balancer.needsReassignment(anyObject())).andReturn(false);
+ replay(balancer);
+ return balancer;
+ }
+ };
+ tls.init(environment);
+
+ assertFalse(tls.needsReassignment(new TestCurrAssignment(tid1, "G1")));
+ assertTrue(tls.needsReassignment(new TestCurrAssignment(tid1, "G2")));
+
+ assertFalse(tls.needsReassignment(new TestCurrAssignment(tid2, "G2")));
+ assertTrue(tls.needsReassignment(new TestCurrAssignment(tid2, "G1")));
+
+ assertFalse(
+ tls.needsReassignment(new TestCurrAssignment(tid3,
Constants.DEFAULT_RESOURCE_GROUP_NAME)));
+ assertTrue(tls.needsReassignment(new TestCurrAssignment(tid3, "G1")));
+
+ // test when the delegated table balancer returns true for one table and
false for others
+ var tls2 = new TableLoadBalancer() {
+ @Override
+ protected TabletBalancer getBalancerForTable(TableId tableId) {
+ TabletBalancer balancer = createMock(TabletBalancer.class);
+
expect(balancer.needsReassignment(anyObject())).andReturn(tableId.equals(tid1));
+ replay(balancer);
+ return balancer;
+ }
+ };
+ tls2.init(environment);
+
+ assertTrue(tls2.needsReassignment(new TestCurrAssignment(tid1, "G1")));
+ assertTrue(tls2.needsReassignment(new TestCurrAssignment(tid1, "G2")));
+
+ assertFalse(tls2.needsReassignment(new TestCurrAssignment(tid2, "G2")));
+ assertTrue(tls2.needsReassignment(new TestCurrAssignment(tid2, "G1")));
+
+ assertFalse(tls2
+ .needsReassignment(new TestCurrAssignment(tid3,
Constants.DEFAULT_RESOURCE_GROUP_NAME)));
+ assertTrue(tls2.needsReassignment(new TestCurrAssignment(tid3, "G1")));
+ }
+
}
diff --cc
core/src/test/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlannerTest.java
index 8811360b6c,e0c32dde6f..6f14f6eab9
---
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlannerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlannerTest.java
@@@ -36,22 -34,18 +36,18 @@@ import java.util.List
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
- import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
- import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
- import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
import org.apache.accumulo.core.spi.common.ServiceEnvironment.Configuration;
import org.apache.accumulo.core.spi.compaction.CompactionPlan.Builder;
import
org.apache.accumulo.core.spi.compaction.CompactionPlanner.InitParameters;
- import org.apache.accumulo.core.util.ConfigurationImpl;
-import org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.accumulo.core.util.compaction.CompactionJobImpl;
+import org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer;
import org.apache.accumulo.core.util.compaction.CompactionPlanImpl;
import org.apache.accumulo.core.util.compaction.CompactionPlannerInitParams;
import org.easymock.EasyMock;
@@@ -181,15 -177,16 +177,14 @@@ public class RatioBasedCompactionPlanne
}
@Test
- public void testUserCompaction() throws Exception {
- ServiceEnvironment.Configuration config = ServiceEnvironment.Configuration
+ public void testUserCompaction() {
- ConfigurationCopy aconf = new
ConfigurationCopy(DefaultConfiguration.getInstance());
- aconf.set(prefix + "cs1.planner.opts.maxOpen", "15");
- ConfigurationImpl config = new ConfigurationImpl(aconf);
++ var config = ServiceEnvironment.Configuration
+ .from(Map.of(prefix + "cs1.planner.opts.maxOpen", "15"), true);
- String executors = "[{'name':'small','type':
'internal','maxSize':'32M','numThreads':1},"
- + "{'name':'medium','type':
'internal','maxSize':'128M','numThreads':2},"
- + "{'name':'large','type':
'internal','maxSize':'512M','numThreads':3},"
- + "{'name':'huge','type': 'internal','numThreads':4}]";
+ String groups = "[{'group':'small','maxSize':'32M'},
{'group':'medium','maxSize':'128M'},"
+ + "{'group':'large','maxSize':'512M'}, {'group':'huge'}]";
- var planner = createPlanner(config, executors);
+ var planner = createPlanner(config, groups);
var all = createCFs("F1", "3M", "F2", "3M", "F3", "11M", "F4", "12M",
"F5", "13M");
var candidates = createCFs("F3", "11M", "F4", "12M", "F5", "13M");
var compacting =
@@@ -544,10 -453,10 +539,10 @@@
Map<String,String> overrides = new HashMap<>();
overrides.put(Property.COMPACTION_SERVICE_PREFIX.getKey() +
"cs1.planner.opts.maxOpen", "10");
overrides.put(Property.TABLE_FILE_MAX.getKey(), "7");
- var conf = new
ConfigurationImpl(SiteConfiguration.empty().withOverrides(overrides).build());
+ var conf = ServiceEnvironment.Configuration.from(overrides, false);
// For this case need to compact three files and the highest ratio that
achieves that is 1.8
- var planner = createPlanner(conf, executors);
+ var planner = createPlanner(conf, groups);
var all = createCFs(1000, 1.1, 1.9, 1.8, 1.6, 1.3, 1.4, 1.3, 1.2, 1.1);
var params = createPlanningParams(all, all, Set.of(), 3,
CompactionKind.SYSTEM, conf);
var plan = planner.makePlan(params);
@@@ -620,10 -530,10 +615,10 @@@
Map<String,String> overrides = new HashMap<>();
overrides.put(Property.COMPACTION_SERVICE_PREFIX.getKey() +
"cs1.planner.opts.maxOpen", "10");
overrides.put(Property.TABLE_FILE_MAX.getKey(), "7");
- var conf = new
ConfigurationImpl(SiteConfiguration.empty().withOverrides(overrides).build());
+ var conf = ServiceEnvironment.Configuration.from(overrides, false);
// ensure that when a compaction would be over the max size limit that it
is not planned
- var planner = createPlanner(conf, executors);
+ var planner = createPlanner(conf, groups);
var all = createCFs(1_000_000_000, 2, 2, 2, 2, 2, 2, 2);
var params = createPlanningParams(all, all, Set.of(), 3,
CompactionKind.SYSTEM, conf);
var plan = planner.makePlan(params);
@@@ -658,9 -569,9 +653,9 @@@
overrides.put(Property.COMPACTION_SERVICE_PREFIX.getKey() +
"cs1.planner.opts.maxOpen", "10");
overrides.put(Property.TABLE_FILE_MAX.getKey(), "0");
overrides.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "5");
- var conf = new
ConfigurationImpl(SiteConfiguration.empty().withOverrides(overrides).build());
+ var conf = ServiceEnvironment.Configuration.from(overrides, false);
- var planner = createPlanner(conf, executors);
+ var planner = createPlanner(conf, groups);
var all = createCFs(1000, 1.9, 1.8, 1.7, 1.6, 1.5, 1.4, 1.3, 1.2, 1.1);
var params = createPlanningParams(all, all, Set.of(), 3,
CompactionKind.SYSTEM, conf);
var plan = planner.makePlan(params);
