Repository: accumulo Updated Branches: refs/heads/master a45dbfbcf -> 77cd4cc7d
ACCUMULO-3623 wait for stats to catch up to master state Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/77cd4cc7 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/77cd4cc7 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/77cd4cc7 Branch: refs/heads/master Commit: 77cd4cc7dbd863c075eab006ce45a4e675258873 Parents: a45dbfb Author: Eric C. Newton <eric.new...@gmail.com> Authored: Wed Feb 25 12:53:25 2015 -0500 Committer: Eric C. Newton <eric.new...@gmail.com> Committed: Wed Feb 25 12:53:25 2015 -0500 ---------------------------------------------------------------------- .../java/org/apache/accumulo/master/Master.java | 32 +++++++++++++++++--- .../accumulo/master/TabletGroupWatcher.java | 11 ++++++- .../accumulo/master/state/TableStats.java | 9 +++++- 3 files changed, 45 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/77cd4cc7/server/master/src/main/java/org/apache/accumulo/master/Master.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java index 16cdd54..5e6dcfb 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@ -856,6 +856,28 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List private class StatusThread extends Daemon { + private boolean goodStats() { + int start; + switch (getMasterState()) { + case UNLOAD_METADATA_TABLETS: + start = 1; + break; + case UNLOAD_ROOT_TABLET: + start = 2; + break; + default: + start = 0; + } + for (int i = start; i < watchers.size(); i++) { + TabletGroupWatcher watcher = watchers.get(i); + if (watcher.stats.getLastMasterState() != getMasterState()) { + log.debug(watcher.getName() + ": " + watcher.stats.getLastMasterState() + " != " + getMasterState()); + return false; + } + } + return true; + } + @Override public void run() { setName("Status Thread"); @@ -883,27 +905,27 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List case SAFE_MODE: { int count = nonMetaDataTabletsAssignedOrHosted(); log.debug(String.format("There are %d non-metadata tablets assigned or hosted", count)); - if (count == 0) + if (count == 0 && goodStats()) setMasterState(MasterState.UNLOAD_METADATA_TABLETS); } break; case UNLOAD_METADATA_TABLETS: { int count = assignedOrHosted(METADATA_TABLE_ID); log.debug(String.format("There are %d metadata tablets assigned or hosted", count)); - if (count == 0) + if (count == 0 && goodStats()) setMasterState(MasterState.UNLOAD_ROOT_TABLET); } break; case UNLOAD_ROOT_TABLET: { int count = assignedOrHosted(METADATA_TABLE_ID); - if (count > 0) { + if (count > 0 && goodStats()) { log.debug(String.format("%d metadata tablets online", count)); setMasterState(MasterState.UNLOAD_ROOT_TABLET); } int root_count = assignedOrHosted(ROOT_TABLE_ID); - if (root_count > 0) + if (root_count > 0 && goodStats()) log.debug("The root tablet is still assigned or hosted"); - if (count + root_count == 0) { + if (count + root_count == 0 && goodStats()) { Set<TServerInstance> currentServers = tserverSet.getCurrentServers(); log.debug("stopping " + currentServers.size() + " tablet servers"); for (TServerInstance server : currentServers) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/77cd4cc7/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java index 35a2d10..755e322 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java +++ b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java @@ -46,6 +46,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.master.thrift.MasterState; import org.apache.accumulo.core.master.thrift.TabletServerStatus; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; @@ -94,6 +95,7 @@ class TabletGroupWatcher extends Daemon { private final Master master; final TabletStateStore store; final TabletGroupWatcher dependentWatcher; + private MasterState masterState; final TableStats stats = new TableStats(); @@ -107,6 +109,11 @@ class TabletGroupWatcher extends Daemon { return stats.getLast(); } + // returns the master state under which stats were collected + MasterState statsState() { + return masterState; + } + TableCounts getStats(Text tableId) { return stats.getLast(tableId); } @@ -121,6 +128,7 @@ class TabletGroupWatcher extends Daemon { while (this.master.stillMaster()) { // slow things down a little, otherwise we spam the logs when there are many wake-up events UtilWaitThread.sleep(100); + masterState = master.getMasterState(); int totalUnloaded = 0; int unloaded = 0; @@ -154,6 +162,7 @@ class TabletGroupWatcher extends Daemon { List<TabletLocationState> assignedToDeadServers = new ArrayList<TabletLocationState>(); Map<KeyExtent,TServerInstance> unassigned = new HashMap<KeyExtent,TServerInstance>(); + MasterState masterState = master.getMasterState(); int[] counts = new int[TabletState.values().length]; stats.begin(); // Walk through the tablets in our store, and work tablets @@ -279,7 +288,7 @@ class TabletGroupWatcher extends Daemon { flushChanges(destinations, assignments, assigned, assignedToDeadServers, unassigned); // provide stats after flushing changes to avoid race conditions w/ delete table - stats.end(); + stats.end(masterState); // Report changes for (TabletState state : TabletState.values()) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/77cd4cc7/server/master/src/main/java/org/apache/accumulo/master/state/TableStats.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/state/TableStats.java b/server/master/src/main/java/org/apache/accumulo/master/state/TableStats.java index 127406c..8d87896 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/state/TableStats.java +++ b/server/master/src/main/java/org/apache/accumulo/master/state/TableStats.java @@ -19,6 +19,7 @@ package org.apache.accumulo.master.state; import java.util.HashMap; import java.util.Map; +import org.apache.accumulo.core.master.thrift.MasterState; import org.apache.accumulo.server.master.state.TabletState; import org.apache.hadoop.io.Text; @@ -27,6 +28,7 @@ public class TableStats { private Map<Text,TableCounts> next; private long startScan = 0; private long endScan = 0; + private MasterState state; public synchronized void begin() { next = new HashMap<Text,TableCounts>(); @@ -42,16 +44,21 @@ public class TableStats { counts.counts[state.ordinal()]++; } - public synchronized void end() { + public synchronized void end(MasterState state) { last = next; next = null; endScan = System.currentTimeMillis(); + this.state = state; } public synchronized Map<Text,TableCounts> getLast() { return last; } + public synchronized MasterState getLastMasterState() { + return state; + } + public synchronized TableCounts getLast(Text tableId) { TableCounts result = last.get(tableId); if (result == null)
