This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 8c3812bb3c Created HostRegexTableLoadBalancer test default pool, fixed
bug (#5492)
8c3812bb3c is described below
commit 8c3812bb3ceec92cff6ecb8a77f935c156397199
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Tue Apr 22 11:23:16 2025 -0400
Created HostRegexTableLoadBalancer test default pool, fixed bug (#5492)
Users can define a regex for the default pool. The code already supports
this, but we didn't have tests for it. Also fixed a bug in the balance
method
with the out of bounds check. The code was using `Map.getOrDefault`
which returns the default value, but doesn't insert it into the map. The
side effect of this is that the out of bounds checks were not being
performed
because the condition could never be true. Modified the code to use
`Map.computeIfAbsent` which will insert the value into the map for
subsequent checks.
Closes #5489
---
.../spi/balancer/HostRegexTableLoadBalancer.java | 5 +-
.../balancer/HostRegexTableLoadBalancerTest.java | 174 +++++++++++++++++++++
2 files changed, 176 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancer.java
b/core/src/main/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancer.java
index b64863fee1..aad49dc238 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancer.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancer.java
@@ -395,9 +395,8 @@ public class HostRegexTableLoadBalancer extends
TableLoadBalancer {
Map<String,SortedMap<TabletServerId,TServerStatus>> currentGrouped =
splitCurrentByRegex(params.currentStatus());
- if ((now
- - this.lastOOBCheckTimes.getOrDefault(params.partitionName(),
System.currentTimeMillis()))
- > myConf.oobCheckMillis) {
+ if ((now - this.lastOOBCheckTimes.computeIfAbsent(params.partitionName(),
+ (p) -> System.currentTimeMillis())) > myConf.oobCheckMillis) {
try {
// Check to see if a tablet is assigned outside the bounds of the
pool. If so, migrate it.
for (String table : tableIdMap.keySet()) {
diff --git
a/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java
b/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java
index 9c9de172c2..4af9708a03 100644
---
a/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java
@@ -186,6 +186,7 @@ public class HostRegexTableLoadBalancerTest extends
BaseHostRegexTableLoadBalanc
barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997,
Integer.toHexString(1))));
assertTrue(
barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997,
Integer.toHexString(1))));
+ // confirms that the default pool contains un-assigned tservers
assertTrue(groups.containsKey(DEFAULT_POOL));
SortedMap<TabletServerId,TServerStatus> defHosts =
groups.get(DEFAULT_POOL);
assertEquals(5, defHosts.size());
@@ -201,6 +202,179 @@ public class HostRegexTableLoadBalancerTest extends
BaseHostRegexTableLoadBalanc
defHosts.containsKey(new TabletServerIdImpl("192.168.0.15", 9997,
Integer.toHexString(1))));
}
+ @Test
+ public void testDefaultPoolAll() {
+ // If all tservers are included in regular expressions, the the default
pool
+ // contains all tservers
+ HashMap<String,String> props = new HashMap<>(DEFAULT_TABLE_PROPERTIES);
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX +
FOO.getTableName(), "r01.*");
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX +
BAR.getTableName(), "r02.*");
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX +
BAZ.getTableName(), "r03.*");
+ init(props);
+ Map<String,SortedMap<TabletServerId,TServerStatus>> groups =
+ this.splitCurrentByRegex(createCurrent(15));
+ assertEquals(4, groups.size());
+ assertTrue(groups.containsKey(FOO.getTableName()));
+ SortedMap<TabletServerId,TServerStatus> fooHosts =
groups.get(FOO.getTableName());
+ assertEquals(5, fooHosts.size());
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997,
Integer.toHexString(1))));
+ assertTrue(groups.containsKey(BAR.getTableName()));
+ SortedMap<TabletServerId,TServerStatus> barHosts =
groups.get(BAR.getTableName());
+ assertEquals(5, barHosts.size());
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997,
Integer.toHexString(1))));
+ assertTrue(groups.containsKey(BAZ.getTableName()));
+ SortedMap<TabletServerId,TServerStatus> bazHosts =
groups.get(BAZ.getTableName());
+ assertEquals(5, bazHosts.size());
+ assertTrue(
+ bazHosts.containsKey(new TabletServerIdImpl("192.168.0.11", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ bazHosts.containsKey(new TabletServerIdImpl("192.168.0.12", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ bazHosts.containsKey(new TabletServerIdImpl("192.168.0.13", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ bazHosts.containsKey(new TabletServerIdImpl("192.168.0.14", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ bazHosts.containsKey(new TabletServerIdImpl("192.168.0.15", 9997,
Integer.toHexString(1))));
+ assertTrue(groups.containsKey(DEFAULT_POOL));
+ SortedMap<TabletServerId,TServerStatus> defHosts =
groups.get(DEFAULT_POOL);
+ assertEquals(15, defHosts.size());
+ }
+
+ @Test
+ public void testSplitCurrentByRegexDefineDefaultPoolOverlapping() {
+ HashMap<String,String> props = new HashMap<>(DEFAULT_TABLE_PROPERTIES);
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX +
FOO.getTableName(), "r01.*");
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX +
BAR.getTableName(), "r02.*");
+ // Normally the DEFAULT pool would be comprised of the hosts not included
in a regex.
+ // Here we are going to define it as also being on rack1
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + DEFAULT_POOL,
"r01.*");
+ init(props);
+ Map<String,SortedMap<TabletServerId,TServerStatus>> groups =
+ this.splitCurrentByRegex(createCurrent(15));
+ assertEquals(3, groups.size());
+ assertTrue(groups.containsKey(FOO.getTableName()));
+ SortedMap<TabletServerId,TServerStatus> fooHosts =
groups.get(FOO.getTableName());
+ assertEquals(5, fooHosts.size());
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997,
Integer.toHexString(1))));
+ assertTrue(groups.containsKey(BAR.getTableName()));
+ SortedMap<TabletServerId,TServerStatus> barHosts =
groups.get(BAR.getTableName());
+ assertEquals(5, barHosts.size());
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997,
Integer.toHexString(1))));
+ assertTrue(groups.containsKey(DEFAULT_POOL));
+ SortedMap<TabletServerId,TServerStatus> defHosts =
groups.get(DEFAULT_POOL);
+ assertEquals(10, defHosts.size());
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.11", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.12", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.13", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.14", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.15", 9997,
Integer.toHexString(1))));
+ }
+
+ @Test
+ public void testSplitCurrentByRegexDefineDefaultPoolNonOverlapping() {
+ HashMap<String,String> props = new HashMap<>(DEFAULT_TABLE_PROPERTIES);
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX +
FOO.getTableName(), "r01.*");
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX +
BAR.getTableName(), "r02.*");
+ // Normally the DEFAULT pool would be comprised of the hosts not included
in a regex.
+ // Here we are going to define it as also being on rack3
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + DEFAULT_POOL,
"r03.*");
+ init(props);
+ Map<String,SortedMap<TabletServerId,TServerStatus>> groups =
+ this.splitCurrentByRegex(createCurrent(15));
+ assertEquals(3, groups.size());
+ assertTrue(groups.containsKey(FOO.getTableName()));
+ SortedMap<TabletServerId,TServerStatus> fooHosts =
groups.get(FOO.getTableName());
+ assertEquals(5, fooHosts.size());
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ fooHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997,
Integer.toHexString(1))));
+ assertTrue(groups.containsKey(BAR.getTableName()));
+ SortedMap<TabletServerId,TServerStatus> barHosts =
groups.get(BAR.getTableName());
+ assertEquals(5, barHosts.size());
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997,
Integer.toHexString(1))));
+ assertTrue(groups.containsKey(DEFAULT_POOL));
+ SortedMap<TabletServerId,TServerStatus> defHosts =
groups.get(DEFAULT_POOL);
+ assertEquals(5, defHosts.size());
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.11", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.12", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.13", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.14", 9997,
Integer.toHexString(1))));
+ assertTrue(
+ defHosts.containsKey(new TabletServerIdImpl("192.168.0.15", 9997,
Integer.toHexString(1))));
+
+ }
+
@Test
public void testSplitCurrentByRegexUsingOverlappingPools() {
HashMap<String,String> props = new HashMap<>(DEFAULT_TABLE_PROPERTIES);
