Repository: accumulo Updated Branches: refs/heads/ACCUMULO-4173 796cbcf7f -> 47679074f
Addressed some comments about using the same comparator as the current set of tservers and allowing multiple pools per host Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/47679074 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/47679074 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/47679074 Branch: refs/heads/ACCUMULO-4173 Commit: 47679074f1ef784708d1aab86f292c64dcd99178 Parents: 796cbcf Author: Dave Marion <dlmar...@apache.org> Authored: Wed Mar 30 14:06:43 2016 -0400 Committer: Dave Marion <dlmar...@apache.org> Committed: Wed Mar 30 14:06:43 2016 -0400 ---------------------------------------------------------------------- .../balancer/HostRegexTableLoadBalancer.java | 78 ++++++++------------ .../HostRegexTableLoadBalancerTest.java | 40 ++++++---- 2 files changed, 54 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/47679074/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java index 8e28132..74154a5 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java @@ -91,32 +91,12 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer { if ((System.currentTimeMillis() - lastPoolRecheck) > poolRecheckMillis) { Map<String,SortedMap<TServerInstance,TabletServerStatus>> newPools = new HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>(); for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) { - String tableName = getPoolNameForHost(e.getKey().host()); - if (!newPools.containsKey(tableName)) { - newPools.put(tableName, new TreeMap<TServerInstance,TabletServerStatus>()); - } - newPools.get(tableName).put(e.getKey(), e.getValue()); - } - // Ensure that no host is in more than one pool - // TODO: I'm not sure that I need to check for disjoint as the call to getPoolNameForHost checks for more than one match - boolean error = false; - for (SortedMap<TServerInstance,TabletServerStatus> s1 : newPools.values()) { - for (SortedMap<TServerInstance,TabletServerStatus> s2 : newPools.values()) { - if (s1 == s2) { - continue; - } - if (!Collections.disjoint(s1.keySet(), s2.keySet())) { - LOG.error("Pools are not disjoint: {}, there is a problem with your regular expressions. Putting all servers in default pool", newPools); - error = true; + List<String> poolNames = getPoolNamesForHost(e.getKey().host()); + for (String pool : poolNames) { + if (!newPools.containsKey(pool)) { + newPools.put(pool, new TreeMap<TServerInstance,TabletServerStatus>(current.comparator())); } - } - } - if (error) { - // Put all servers into the default pool - newPools.clear(); - newPools.put(DEFAULT_POOL, new TreeMap<TServerInstance,TabletServerStatus>()); - for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) { - newPools.get(DEFAULT_POOL).put(e.getKey(), e.getValue()); + newPools.get(pool).put(e.getKey(), e.getValue()); } } pools = newPools; @@ -126,33 +106,32 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer { } /** - * Matches host against the regexes and returns the matching pool name + * Matches host against the regexes and returns the matching pool names * * @param host * tablet server host - * @return name of pool. will return default pool if host matches more than one regex + * @return pool names, will return default pool if host matches more no regex */ - protected String getPoolNameForHost(String host) { + protected List<String> getPoolNamesForHost(String host) { String test = host; - String table = DEFAULT_POOL; if (!isIpBasedRegex) { try { test = getNameFromIp(host); } catch (UnknownHostException e1) { LOG.error("Unable to determine host name for IP: " + host + ", setting to default pool", e1); - return table; + return Collections.singletonList(DEFAULT_POOL); } } + List<String> pools = new ArrayList<>(); for (Entry<String,Pattern> e : poolNameToRegexPattern.entrySet()) { if (e.getValue().matcher(test).matches()) { - if (!table.equals(DEFAULT_POOL)) { - LOG.warn("host {} matches more than one regex, assigning to default pool", host); - return DEFAULT_POOL; - } - table = e.getKey(); + pools.add(e.getKey()); } } - return table; + if (pools.size() == 0) { + pools.add(DEFAULT_POOL); + } + return pools; } protected String getNameFromIp(String hostIp) throws UnknownHostException { @@ -240,20 +219,21 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer { if ((System.currentTimeMillis() - this.lastOOBCheck) > this.oobCheckMillis) { // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it. for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) { - String assignedPool = getPoolNameForHost(e.getKey().host()); - for (String pool : poolNameToRegexPattern.keySet()) { - if (assignedPool.equals(pool) || pool.equals(DEFAULT_POOL)) { - continue; - } - String tid = getTableOperations().tableIdMap().get(pool); - try { - List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), tid); - for (TabletStats ts : outOfBoundsTablets) { - LOG.info("Tablet {} is currently outside the bounds of the regex, reassigning", ts.toString()); - unassigned.put(new KeyExtent(ts.getExtent()), e.getKey()); + for (String assignedPool : getPoolNamesForHost(e.getKey().host())) { + for (String pool : poolNameToRegexPattern.keySet()) { + if (assignedPool.equals(pool) || pool.equals(DEFAULT_POOL)) { + continue; + } + String tid = getTableOperations().tableIdMap().get(pool); + try { + List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), tid); + for (TabletStats ts : outOfBoundsTablets) { + LOG.info("Tablet {} is currently outside the bounds of the regex, reassigning", ts.toString()); + unassigned.put(new KeyExtent(ts.getExtent()), e.getKey()); + } + } catch (TException e1) { + LOG.error("Error in OOB check getting tablets for table {} from server {}", tid, e.getKey().host(), e); } - } catch (TException e1) { - LOG.error("Error in OOB check getting tablets for table {} from server {}", tid, e.getKey().host(), e); } } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/47679074/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java index e14b511..c33752a 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java @@ -344,7 +344,7 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer { } @Test - public void testSplitCurrentByRegexUsingHostnameMatchesTooMany() { + public void testSplitCurrentByRegexUsingOverlappingPools() { init((ServerConfiguration) new TestServerConfigurationFactory(instance) { @Override public TableConfiguration getTableConfiguration(String tableId) { @@ -375,25 +375,35 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer { Assert.assertEquals(2, groups.size()); Assert.assertTrue(groups.containsKey(FOO.getTableName())); SortedMap<TServerInstance,TabletServerStatus> fooHosts = groups.get(FOO.getTableName()); - Assert.assertEquals(5, fooHosts.size()); + Assert.assertEquals(15, fooHosts.size()); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1))); + Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1))); Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1))); Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1))); Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1))); Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1))); Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1))); - Assert.assertTrue(groups.containsKey(DEFAULT_POOL)); - SortedMap<TServerInstance,TabletServerStatus> defHosts = groups.get(DEFAULT_POOL); - Assert.assertEquals(10, defHosts.size()); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1))); - Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1))); + Assert.assertTrue(groups.containsKey(BAR.getTableName())); + SortedMap<TServerInstance,TabletServerStatus> barHosts = groups.get(BAR.getTableName()); + Assert.assertEquals(10, barHosts.size()); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1))); + Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1))); } @Test
