Repository: accumulo Updated Branches: refs/heads/1.8 cdafd0203 -> 6c20e50c2
ACCUMULO-4654 Continue balancing even with pending migrations Added the ability to continue balancing with the HostRegexTableLoadBalancer even if there are pending migrations. Also added detection for tables that have been continuously migrating for over an hour. Closes #272 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c20e50c Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c20e50c Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c20e50c Branch: refs/heads/1.8 Commit: 6c20e50c223a44deaccf4e0690ccec9b5749b21c Parents: cdafd02 Author: Ivan Bella <i...@bella.name> Authored: Mon Jun 19 13:57:44 2017 -0400 Committer: Ivan Bella <i...@bella.name> Committed: Wed Jul 19 17:20:24 2017 -0400 ---------------------------------------------------------------------- .../balancer/HostRegexTableLoadBalancer.java | 117 +++++++++++++++++-- .../BaseHostRegexTableLoadBalancerTest.java | 65 ++++++++++- ...gexTableLoadBalancerReconfigurationTest.java | 2 +- .../HostRegexTableLoadBalancerTest.java | 59 +++++++++- 4 files changed, 224 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java index 5207230..dbf03d0 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java @@ -19,6 +19,7 @@ package org.apache.accumulo.server.master.balancer; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -31,11 +32,15 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.regex.Pattern; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationObserver; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.impl.KeyExtent; +import org.apache.accumulo.core.master.thrift.TableInfo; import org.apache.accumulo.core.master.thrift.TabletServerStatus; import org.apache.accumulo.core.tabletserver.thrift.TabletStats; import org.apache.accumulo.server.conf.ServerConfiguration; @@ -65,7 +70,9 @@ import org.slf4j.LoggerFactory; * <b>table.custom.balancer.host.regex.is.ip=true</b><br> * It's possible that this balancer may create a lot of migrations. To limit the number of migrations that are created during a balance call, set the following * property (default 250):<br> - * <b>table.custom.balancer.host.regex.concurrent.migrations</b> + * <b>table.custom.balancer.host.regex.concurrent.migrations</b> This balancer can continue balancing even if there are outstanding migrations. To limit the + * number of outstanding migrations in which this balancer will continue balancing, set the following property (default 0):<br> + * <b>table.custom.balancer.host.regex.max.outstanding.migrations</b> * */ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements ConfigurationObserver { @@ -81,15 +88,24 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con + "balancer.host.regex.concurrent.migrations"; private static final int HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT = 250; protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL"; + private static final int DEFAULT_OUTSTANDING_MIGRATIONS = 0; + public static final String HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + + "balancer.host.regex.max.outstanding.migrations"; protected long oobCheckMillis = AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT); + private static final long ONE_HOUR = 60 * 60 * 1000; + private static final Set<KeyExtent> EMPTY_MIGRATIONS = Collections.EMPTY_SET; + private Map<String,String> tableIdToTableName = null; private Map<String,Pattern> poolNameToRegexPattern = null; private volatile long lastOOBCheck = System.currentTimeMillis(); - private boolean isIpBasedRegex = false; + private volatile boolean isIpBasedRegex = false; private Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = new HashMap<>(); - private int maxTServerMigrations = HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT; + private volatile int maxTServerMigrations = HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT; + private volatile int maxOutstandingMigrations = DEFAULT_OUTSTANDING_MIGRATIONS; + private final Map<KeyExtent,TabletMigration> migrationsFromLastPass = new HashMap<KeyExtent,TabletMigration>(); + private final Map<String,Long> tableToTimeSinceNoMigrations = new HashMap<String,Long>(); /** * Group the set of current tservers by pool name. Tservers that don't match a regex are put into a default pool. This could be expensive in the terms of the @@ -191,7 +207,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con for (Entry<String,String> customProp : customProps.entrySet()) { if (customProp.getKey().startsWith(HOST_BALANCER_PREFIX)) { if (customProp.getKey().equals(HOST_BALANCER_OOB_CHECK_KEY) || customProp.getKey().equals(HOST_BALANCER_REGEX_USING_IPS_KEY) - || customProp.getKey().equals(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY)) { + || customProp.getKey().equals(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY) || customProp.getKey().equals(HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY)) { continue; } String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length()); @@ -213,6 +229,10 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con if (null != migrations) { maxTServerMigrations = Integer.parseInt(migrations); } + String outstanding = conf.getConfiguration().get(HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY); + if (null != outstanding) { + this.maxOutstandingMigrations = Integer.parseInt(outstanding); + } LOG.info("{}", this); } @@ -234,6 +254,14 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con return poolNameToRegexPattern; } + public int getMaxMigrations() { + return maxTServerMigrations; + } + + public int getMaxOutstandingMigrations() { + return maxOutstandingMigrations; + } + public long getOobCheckMillis() { return oobCheckMillis; } @@ -301,8 +329,10 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con return minBalanceTime; Map<String,String> tableIdMap = t.tableIdMap(); + long now = System.currentTimeMillis(); + Map<String,SortedMap<TServerInstance,TabletServerStatus>> currentGrouped = splitCurrentByRegex(current); - if ((System.currentTimeMillis() - this.lastOOBCheck) > this.oobCheckMillis) { + if ((now - this.lastOOBCheck) > this.oobCheckMillis) { try { // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it. for (String table : t.list()) { @@ -343,7 +373,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con TServerInstance nextTS = iter.next(); LOG.info("Tablet {} is currently outside the bounds of the regex, migrating from {} to {}", ke, e.getKey(), nextTS); migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS)); - if (migrationsOut.size() > this.maxTServerMigrations) { + if (migrationsOut.size() >= this.maxTServerMigrations) { break; } } else { @@ -356,6 +386,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con } } } finally { + // this could have taken a while...get a new time this.lastOOBCheck = System.currentTimeMillis(); } } @@ -367,8 +398,34 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con } if (migrations != null && migrations.size() > 0) { - LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size()); - return minBalanceTime; + if (migrations.size() >= maxOutstandingMigrations) { + LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size()); + if (LOG.isTraceEnabled()) { + LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10)); + } + return minBalanceTime; + } + + LOG.debug("Current outstanding migrations of {} being applied", migrations.size()); + if (LOG.isTraceEnabled()) { + LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10)); + } + migrationsFromLastPass.keySet().retainAll(migrations); + SortedMap<TServerInstance,TabletServerStatus> currentCopy = new TreeMap(current); + Multimap<TServerInstance,String> serverTableIdCopied = HashMultimap.create(); + for (TabletMigration migration : migrationsFromLastPass.values()) { + TableInfo fromInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.tablet.getTableId().toString(), migration.oldServer); + if (fromInfo != null) { + fromInfo.setOnlineTablets(fromInfo.getOnlineTablets() - 1); + } + TableInfo toInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.tablet.getTableId().toString(), migration.newServer); + if (toInfo != null) { + toInfo.setOnlineTablets(toInfo.getOnlineTablets() + 1); + } + } + migrations = EMPTY_MIGRATIONS; + } else { + migrationsFromLastPass.clear(); } for (String s : tableIdMap.values()) { @@ -382,15 +439,57 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer implements Con ArrayList<TabletMigration> newMigrations = new ArrayList<>(); getBalancerForTable(s).balance(currentView, migrations, newMigrations); + if (newMigrations.isEmpty()) { + tableToTimeSinceNoMigrations.remove(s); + } else if (tableToTimeSinceNoMigrations.containsKey(s)) { + if ((now - tableToTimeSinceNoMigrations.get(s)) > ONE_HOUR) { + LOG.warn("We have been consistently producing migrations for {}: {}", tableName, Iterables.limit(newMigrations, 10)); + } + } else { + tableToTimeSinceNoMigrations.put(s, now); + } + migrationsOut.addAll(newMigrations); - if (migrationsOut.size() > this.maxTServerMigrations) { + if (migrationsOut.size() >= this.maxTServerMigrations) { break; } } + + for (TabletMigration migration : migrationsOut) { + migrationsFromLastPass.put(migration.tablet, migration); + } + LOG.info("Migrating tablets for balance: {}", migrationsOut); return minBalanceTime; } + /** + * Get a mutable table info for the specified table and server + */ + private TableInfo getTableInfo(SortedMap<TServerInstance,TabletServerStatus> currentCopy, Multimap<TServerInstance,String> serverTableIdCopied, + String tableId, TServerInstance server) { + TableInfo newInfo = null; + if (currentCopy.containsKey(server)) { + Map<String,TableInfo> newTableMap = currentCopy.get(server).getTableMap(); + if (newTableMap != null) { + newInfo = newTableMap.get(tableId); + if (newInfo != null) { + Collection<String> tableIdCopied = serverTableIdCopied.get(server); + if (tableIdCopied.isEmpty()) { + newTableMap = new HashMap<String,TableInfo>(newTableMap); + currentCopy.get(server).setTableMap(newTableMap); + } + if (!tableIdCopied.contains(tableId)) { + newInfo = new TableInfo(newInfo); + newTableMap.put(tableId, newInfo); + tableIdCopied.add(tableId); + } + } + } + } + return newInfo; + } + @Override public void propertyChanged(String key) { parseConfiguration(this.configuration); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java index a1730cb..e44491d 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java @@ -35,16 +35,20 @@ import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.impl.ClientContext; import org.apache.accumulo.core.client.impl.TableOperationsImpl; +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.impl.KeyExtent; +import org.apache.accumulo.core.master.thrift.TableInfo; import org.apache.accumulo.core.master.thrift.TabletServerStatus; +import org.apache.accumulo.core.tabletserver.thrift.TabletStats; import org.apache.accumulo.server.conf.ServerConfigurationFactory; import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.hadoop.io.Text; +import org.apache.thrift.TException; import org.easymock.EasyMock; import com.google.common.base.Predicate; @@ -138,9 +142,12 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL protected static final HashMap<String,String> DEFAULT_TABLE_PROPERTIES = new HashMap<>(); { - DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "2s"); + DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "7s"); + DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY, "4"); + DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY, "10"); DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r01.*"); DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r02.*"); + DEFAULT_TABLE_PROPERTIES.put(Property.TABLE_LOAD_BALANCER.getKey(), TestDefaultBalancer.class.getName()); } protected static class TestServerConfigurationFactory extends ServerConfigurationFactory { @@ -155,7 +162,7 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL } @Override - public TableConfiguration getTableConfiguration(String tableId) { + public TableConfiguration getTableConfiguration(final String tableId) { return new TableConfiguration(getInstance(), tableId, null) { @Override public String get(Property property) { @@ -174,6 +181,24 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL } } + protected class TestDefaultBalancer extends DefaultLoadBalancer { + @Override + public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException { + String tableName = idToTableName(tableId); + TServerInstance initialLocation = initialTableLocation.get(tableName); + if (tserver.equals(initialLocation)) { + List<TabletStats> list = new ArrayList<TabletStats>(5); + for (KeyExtent extent : tableExtents.get(tableName)) { + TabletStats stats = new TabletStats(); + stats.setExtent(extent.toThrift()); + list.add(stats); + } + return list; + } + return null; + } + } + protected static final Table FOO = new Table("foo", "1"); protected static final Table BAR = new Table("bar", "2"); protected static final Table BAZ = new Table("baz", "3"); @@ -183,6 +208,7 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL protected final Map<String,String> servers = new HashMap<>(15); protected final SortedMap<TServerInstance,TabletServerStatus> allTabletServers = new TreeMap<>(); protected final Map<String,List<KeyExtent>> tableExtents = new HashMap<>(3); + protected final Map<String,TServerInstance> initialTableLocation = new HashMap<>(3); { servers.put("192.168.0.1", "r01s01"); @@ -217,6 +243,10 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL allTabletServers.put(new TServerInstance("192.168.0.14:9997", 1), new TabletServerStatus()); allTabletServers.put(new TServerInstance("192.168.0.15:9997", 1), new TabletServerStatus()); + initialTableLocation.put(FOO.getTableName(), new TServerInstance("192.168.0.1:9997", 1)); + initialTableLocation.put(BAR.getTableName(), new TServerInstance("192.168.0.6:9997", 1)); + initialTableLocation.put(BAZ.getTableName(), new TServerInstance("192.168.0.11:9997", 1)); + tableExtents.put(FOO.getTableName(), new ArrayList<KeyExtent>()); tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("1"), new Text("0"))); tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("2"), new Text("1"))); @@ -235,6 +265,7 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("23"), new Text("22"))); tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("24"), new Text("23"))); tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("25"), new Text("24"))); + } protected boolean tabletInBounds(KeyExtent ke, TServerInstance tsi) { @@ -255,6 +286,18 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL } } + protected String idToTableName(String id) { + if (id.equals(FOO.getId())) { + return FOO.getTableName(); + } else if (id.equals(BAR.getId())) { + return BAR.getTableName(); + } else if (id.equals(BAZ.getId())) { + return BAZ.getTableName(); + } else { + return null; + } + } + @Override protected TableOperations getTableOperations() { return new TableOperationsImpl(EasyMock.createMock(ClientContext.class)) { @@ -280,7 +323,7 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL @Override protected TabletBalancer getBalancerForTable(String table) { - return new DefaultLoadBalancer(); + return new TestDefaultBalancer(); } @Override @@ -296,7 +339,21 @@ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableL String base = "192.168.0."; TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>(); for (int i = 1; i <= numTservers; i++) { - current.put(new TServerInstance(base + i + ":9997", 1), new TabletServerStatus()); + TabletServerStatus status = new TabletServerStatus(); + Map<String,TableInfo> tableMap = new HashMap<String,TableInfo>(); + tableMap.put(FOO.getId(), new TableInfo()); + tableMap.put(BAR.getId(), new TableInfo()); + tableMap.put(BAZ.getId(), new TableInfo()); + status.setTableMap(tableMap); + current.put(new TServerInstance(base + i + ":9997", 1), status); + } + // now put all of the tablets on one server + for (Map.Entry<String,TServerInstance> entry : initialTableLocation.entrySet()) { + TabletServerStatus status = current.get(entry.getValue()); + if (status != null) { + String tableId = getTableOperations().tableIdMap().get(entry.getKey()); + status.getTableMap().get(tableId).setOnlineTablets(5); + } } return current; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java index 14f6c3e..1be948f 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java @@ -78,7 +78,7 @@ public class HostRegexTableLoadBalancerReconfigurationTest extends BaseHostRegex DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*"); this.propertiesChanged(); // Wait to trigger the out of bounds check and the repool check - UtilWaitThread.sleep(3000); + UtilWaitThread.sleep(10000); this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut); Assert.assertEquals(5, migrationsOut.size()); for (TabletMigration migration : migrationsOut) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java index 24054f5..120aab9 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java @@ -36,6 +36,7 @@ import org.apache.accumulo.core.data.thrift.TKeyExtent; import org.apache.accumulo.core.master.thrift.TabletServerStatus; import org.apache.accumulo.core.tabletserver.thrift.TabletStats; import org.apache.accumulo.fate.util.UtilWaitThread; +import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.master.state.TabletMigration; @@ -50,10 +51,12 @@ public class HostRegexTableLoadBalancerTest extends BaseHostRegexTableLoadBalanc @Test public void testInit() { init(factory); - Assert.assertEquals("OOB check interval value is incorrect", 2000, this.getOobCheckMillis()); + Assert.assertEquals("OOB check interval value is incorrect", 7000, this.getOobCheckMillis()); @SuppressWarnings("deprecation") long poolRecheckMillis = this.getPoolRecheckMillis(); Assert.assertEquals("Pool check interval value is incorrect", 0, poolRecheckMillis); + Assert.assertEquals("Max migrations is incorrect", 4, this.getMaxMigrations()); + Assert.assertEquals("Max outstanding migrations is incorrect", 10, this.getMaxOutstandingMigrations()); Assert.assertFalse(isIpBasedRegex()); Map<String,Pattern> patterns = this.getPoolNameToRegexPattern(); Assert.assertEquals(2, patterns.size()); @@ -73,12 +76,58 @@ public class HostRegexTableLoadBalancerTest extends BaseHostRegexTableLoadBalanc } @Test - public void testBalanceWithMigrations() { - List<TabletMigration> migrations = new ArrayList<>(); + public void testBalance() { + init((ServerConfiguration) factory); + Set<KeyExtent> migrations = new HashSet<KeyExtent>(); + List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>(); + long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut); + Assert.assertEquals(20000, wait); + // should balance four tablets in one of the tables before reaching max + Assert.assertEquals(4, migrationsOut.size()); + + // now balance again passing in the new migrations + for (TabletMigration m : migrationsOut) { + migrations.add(m.tablet); + } + migrationsOut.clear(); + wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut); + Assert.assertEquals(20000, wait); + // should balance four tablets in one of the other tables before reaching max + Assert.assertEquals(4, migrationsOut.size()); + + // now balance again passing in the new migrations + for (TabletMigration m : migrationsOut) { + migrations.add(m.tablet); + } + migrationsOut.clear(); + wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut); + Assert.assertEquals(20000, wait); + // should balance four tablets in one of the other tables before reaching max + Assert.assertEquals(4, migrationsOut.size()); + + // now balance again passing in the new migrations + for (TabletMigration m : migrationsOut) { + migrations.add(m.tablet); + } + migrationsOut.clear(); + wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut); + Assert.assertEquals(20000, wait); + // no more balancing to do + Assert.assertEquals(0, migrationsOut.size()); + } + + @Test + public void testBalanceWithTooManyOutstandingMigrations() { + List<TabletMigration> migrationsOut = new ArrayList<>(); init(factory); - long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(2)), Collections.singleton(new KeyExtent()), migrations); + // lets say we already have migrations ongoing for the FOO and BAR table extends (should be 5 of each of them) for a total of 10 + Set<KeyExtent> migrations = new HashSet<KeyExtent>(); + migrations.addAll(tableExtents.get(FOO.getTableName())); + migrations.addAll(tableExtents.get(BAR.getTableName())); + long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut); Assert.assertEquals(20000, wait); - Assert.assertEquals(0, migrations.size()); + // no migrations should have occurred as 10 is the maxOutstandingMigrations + Assert.assertEquals(0, migrationsOut.size()); } @Test
