http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java index 103ba05..58fa9d3 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java @@ -59,34 +59,34 @@ import org.apache.log4j.Logger; import com.google.common.base.Optional; public class FileUtil { - + public static class FileInfo { Key firstKey = new Key(); Key lastKey = new Key(); - + public FileInfo(Key firstKey, Key lastKey) { this.firstKey = firstKey; this.lastKey = lastKey; } - + public Text getFirstRow() { return firstKey.getRow(); } - + public Text getLastRow() { return lastKey.getRow(); } } - + private static final Logger log = Logger.getLogger(FileUtil.class); - + private static Path createTmpDir(AccumuloConfiguration acuConf, VolumeManager fs) throws IOException { - String accumuloDir = fs.choose(Optional.<String>absent(), ServerConstants.getBaseUris()); + String accumuloDir = fs.choose(Optional.<String> absent(), ServerConstants.getBaseUris()); Path result = null; while (result == null) { result = new Path(accumuloDir + Path.SEPARATOR + "tmp/idxReduce_" + String.format("%09d", new Random().nextInt(Integer.MAX_VALUE))); - + try { fs.getFileStatus(result); result = null; @@ -94,9 +94,9 @@ public class FileUtil { } catch (FileNotFoundException fne) { // found an unused temp directory } - + fs.mkdirs(result); - + // try to reserve the tmp dir // In some versions of hadoop, two clients concurrently trying to create the same directory might both return true // Creating a file is not subject to this, so create a special file to make sure we solely will use this directory @@ -105,36 +105,36 @@ public class FileUtil { } return result; } - + public static Collection<String> reduceFiles(AccumuloConfiguration acuConf, Configuration conf, VolumeManager fs, Text prevEndRow, Text endRow, Collection<String> mapFiles, int maxFiles, Path tmpDir, int pass) throws IOException { ArrayList<String> paths = new ArrayList<String>(mapFiles); - + if (paths.size() <= maxFiles) return paths; - + String newDir = String.format("%s/pass_%04d", tmpDir, pass); - + int start = 0; - + ArrayList<String> outFiles = new ArrayList<String>(); - + int count = 0; - + while (start < paths.size()) { int end = Math.min(maxFiles + start, paths.size()); List<String> inFiles = paths.subList(start, end); - + start = end; - + String newMapFile = String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION); - + outFiles.add(newMapFile); FileSystem ns = fs.getVolumeByPath(new Path(newMapFile)).getFileSystem(); FileSKVWriter writer = new RFileOperations().openWriter(newMapFile.toString(), ns, ns.getConf(), acuConf); writer.startDefaultLocalityGroup(); List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size()); - + FileSKVIterator reader = null; try { for (String s : inFiles) { @@ -142,21 +142,21 @@ public class FileUtil { reader = FileOperations.getInstance().openIndex(s, ns, ns.getConf(), acuConf); iters.add(reader); } - + MultiIterator mmfi = new MultiIterator(iters, true); - + while (mmfi.hasTop()) { Key key = mmfi.getTopKey(); - + boolean gtPrevEndRow = prevEndRow == null || key.compareRow(prevEndRow) > 0; boolean lteEndRow = endRow == null || key.compareRow(endRow) <= 0; - + if (gtPrevEndRow && lteEndRow) writer.append(key, new Value(new byte[0])); - + if (!lteEndRow) break; - + mmfi.next(); } } finally { @@ -166,7 +166,7 @@ public class FileUtil { } catch (IOException e) { log.error(e, e); } - + for (SortedKeyValueIterator<Key,Value> r : iters) try { if (r != null) @@ -175,16 +175,16 @@ public class FileUtil { // continue closing log.error(e, e); } - + try { - writer.close(); + writer.close(); } catch (IOException e) { log.error(e, e); throw e; } } } - + return reduceFiles(acuConf, conf, fs, prevEndRow, endRow, outFiles, maxFiles, tmpDir, pass + 1); } @@ -192,114 +192,114 @@ public class FileUtil { double minSplit) throws IOException { return findMidPoint(fs, acuConf, prevEndRow, endRow, mapFiles, minSplit, true); } - + public static double estimatePercentageLTE(VolumeManager fs, AccumuloConfiguration acuconf, Text prevEndRow, Text endRow, Collection<String> mapFiles, Text splitRow) throws IOException { - + Configuration conf = CachedConfiguration.getInstance(); - + Path tmpDir = null; - + int maxToOpen = acuconf.getCount(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN); ArrayList<FileSKVIterator> readers = new ArrayList<FileSKVIterator>(mapFiles.size()); - + try { if (mapFiles.size() > maxToOpen) { tmpDir = createTmpDir(acuconf, fs); - + log.debug("Too many indexes (" + mapFiles.size() + ") to open at once for " + endRow + " " + prevEndRow + ", reducing in tmpDir = " + tmpDir); - + long t1 = System.currentTimeMillis(); mapFiles = reduceFiles(acuconf, conf, fs, prevEndRow, endRow, mapFiles, maxToOpen, tmpDir, 0); long t2 = System.currentTimeMillis(); - + log.debug("Finished reducing indexes for " + endRow + " " + prevEndRow + " in " + String.format("%6.2f secs", (t2 - t1) / 1000.0)); } - + if (prevEndRow == null) prevEndRow = new Text(); - + long numKeys = 0; - + numKeys = countIndexEntries(acuconf, prevEndRow, endRow, mapFiles, true, conf, fs, readers); - + if (numKeys == 0) { // not enough info in the index to answer the question, so instead of going to // the data just punt and return .5 return .5; } - + List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers); MultiIterator mmfi = new MultiIterator(iters, true); - + // skip the prevendrow while (mmfi.hasTop() && mmfi.getTopKey().compareRow(prevEndRow) <= 0) { mmfi.next(); } - + int numLte = 0; - + while (mmfi.hasTop() && mmfi.getTopKey().compareRow(splitRow) <= 0) { numLte++; mmfi.next(); } - + if (numLte > numKeys) { // something went wrong throw new RuntimeException("numLte > numKeys " + numLte + " " + numKeys + " " + prevEndRow + " " + endRow + " " + splitRow + " " + mapFiles); } - + // do not want to return 0% or 100%, so add 1 and 2 below return (numLte + 1) / (double) (numKeys + 2); - + } finally { cleanupIndexOp(acuconf, tmpDir, fs, readers); } } - + /** - * + * * @param mapFiles * - list MapFiles to find the mid point key - * + * * ISSUES : This method used the index files to find the mid point. If the map files have different index intervals this method will not return an * accurate mid point. Also, it would be tricky to use this method in conjunction with an in memory map because the indexing interval is unknown. */ public static SortedMap<Double,Key> findMidPoint(VolumeManager fs, AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<String> mapFiles, double minSplit, boolean useIndex) throws IOException { Configuration conf = CachedConfiguration.getInstance(); - + Collection<String> origMapFiles = mapFiles; - + Path tmpDir = null; - + int maxToOpen = acuConf.getCount(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN); ArrayList<FileSKVIterator> readers = new ArrayList<FileSKVIterator>(mapFiles.size()); - + try { if (mapFiles.size() > maxToOpen) { if (!useIndex) throw new IOException("Cannot find mid point using data files, too many " + mapFiles.size()); tmpDir = createTmpDir(acuConf, fs); - + log.debug("Too many indexes (" + mapFiles.size() + ") to open at once for " + endRow + " " + prevEndRow + ", reducing in tmpDir = " + tmpDir); - + long t1 = System.currentTimeMillis(); mapFiles = reduceFiles(acuConf, conf, fs, prevEndRow, endRow, mapFiles, maxToOpen, tmpDir, 0); long t2 = System.currentTimeMillis(); - + log.debug("Finished reducing indexes for " + endRow + " " + prevEndRow + " in " + String.format("%6.2f secs", (t2 - t1) / 1000.0)); } - + if (prevEndRow == null) prevEndRow = new Text(); - + long t1 = System.currentTimeMillis(); - + long numKeys = 0; - + numKeys = countIndexEntries(acuConf, prevEndRow, endRow, mapFiles, tmpDir == null ? useIndex : false, conf, fs, readers); - + if (numKeys == 0) { if (useIndex) { log.warn("Failed to find mid point using indexes, falling back to data files which is slower. No entries between " + prevEndRow + " and " + endRow @@ -309,48 +309,48 @@ public class FileUtil { } throw new IOException("Failed to find mid point, no entries between " + prevEndRow + " and " + endRow + " for " + mapFiles); } - + List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers); MultiIterator mmfi = new MultiIterator(iters, true); - + // skip the prevendrow while (mmfi.hasTop() && mmfi.getTopKey().compareRow(prevEndRow) <= 0) mmfi.next(); - + // read half of the keys in the index TreeMap<Double,Key> ret = new TreeMap<Double,Key>(); Key lastKey = null; long keysRead = 0; - + Key keyBeforeMidPoint = null; long keyBeforeMidPointPosition = 0; - + while (keysRead < numKeys / 2) { if (lastKey != null && !lastKey.equals(mmfi.getTopKey(), PartialKey.ROW) && (keysRead - 1) / (double) numKeys >= minSplit) { keyBeforeMidPoint = new Key(lastKey); keyBeforeMidPointPosition = keysRead - 1; } - + if (lastKey == null) lastKey = new Key(); - + lastKey.set(mmfi.getTopKey()); - + keysRead++; - + // consume minimum mmfi.next(); } - + if (keyBeforeMidPoint != null) ret.put(keyBeforeMidPointPosition / (double) numKeys, keyBeforeMidPoint); - + long t2 = System.currentTimeMillis(); - + log.debug(String.format("Found midPoint from indexes in %6.2f secs.%n", ((t2 - t1) / 1000.0))); - + ret.put(.5, mmfi.getTopKey()); - + // sanity check for (Key key : ret.values()) { boolean inRange = (key.compareRow(prevEndRow) > 0 && (endRow == null || key.compareRow(endRow) < 1)); @@ -358,7 +358,7 @@ public class FileUtil { throw new IOException("Found mid point is not in range " + key + " " + prevEndRow + " " + endRow + " " + mapFiles); } } - + return ret; } finally { cleanupIndexOp(acuConf, tmpDir, fs, readers); @@ -390,9 +390,9 @@ public class FileUtil { private static long countIndexEntries(AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<String> mapFiles, boolean useIndex, Configuration conf, VolumeManager fs, ArrayList<FileSKVIterator> readers) throws IOException { - + long numKeys = 0; - + // count the total number of index entries for (String ref : mapFiles) { FileSKVIterator reader = null; @@ -402,16 +402,16 @@ public class FileUtil { if (useIndex) reader = FileOperations.getInstance().openIndex(path.toString(), ns, ns.getConf(), acuConf); else - reader = FileOperations.getInstance().openReader(path.toString(), new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false, ns, ns.getConf(), - acuConf); - + reader = FileOperations.getInstance().openReader(path.toString(), new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false, + ns, ns.getConf(), acuConf); + while (reader.hasTop()) { Key key = reader.getTopKey(); if (endRow != null && key.compareRow(endRow) > 0) break; else if (prevEndRow == null || key.compareRow(prevEndRow) > 0) numKeys++; - + reader.next(); } } finally { @@ -422,35 +422,35 @@ public class FileUtil { log.error(e, e); } } - + if (useIndex) readers.add(FileOperations.getInstance().openIndex(path.toString(), ns, ns.getConf(), acuConf)); else - readers.add(FileOperations.getInstance().openReader(path.toString(), new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false, ns, ns.getConf(), - acuConf)); - + readers.add(FileOperations.getInstance().openReader(path.toString(), new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false, + ns, ns.getConf(), acuConf)); + } return numKeys; } - + public static Map<FileRef,FileInfo> tryToGetFirstAndLastRows(VolumeManager fs, AccumuloConfiguration acuConf, Set<FileRef> mapfiles) { - + HashMap<FileRef,FileInfo> mapFilesInfo = new HashMap<FileRef,FileInfo>(); - + long t1 = System.currentTimeMillis(); - + for (FileRef mapfile : mapfiles) { - + FileSKVIterator reader = null; FileSystem ns = fs.getVolumeByPath(mapfile.path()).getFileSystem(); try { reader = FileOperations.getInstance().openReader(mapfile.toString(), false, ns, ns.getConf(), acuConf); - + Key firstKey = reader.getFirstKey(); if (firstKey != null) { mapFilesInfo.put(mapfile, new FileInfo(firstKey, reader.getLastKey())); } - + } catch (IOException ioe) { log.warn("Failed to read map file to determine first and last key : " + mapfile, ioe); } finally { @@ -462,31 +462,31 @@ public class FileUtil { } } } - + } - + long t2 = System.currentTimeMillis(); - + log.debug(String.format("Found first and last keys for %d map files in %6.2f secs", mapfiles.size(), (t2 - t1) / 1000.0)); - + return mapFilesInfo; } - + public static WritableComparable<Key> findLastKey(VolumeManager fs, AccumuloConfiguration acuConf, Collection<FileRef> mapFiles) throws IOException { Key lastKey = null; - + for (FileRef ref : mapFiles) { Path path = ref.path(); FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), acuConf); - + try { if (!reader.hasTop()) // file is empty, so there is no last key continue; - + Key key = reader.getLastKey(); - + if (lastKey == null || key.compareTo(lastKey) > 0) lastKey = key; } finally { @@ -498,41 +498,41 @@ public class FileUtil { } } } - + return lastKey; - + } - + private static class MLong { public MLong(long i) { l = i; } - + long l; } - + public static Map<KeyExtent,Long> estimateSizes(AccumuloConfiguration acuConf, Path mapFile, long fileSize, List<KeyExtent> extents, Configuration conf, VolumeManager fs) throws IOException { - + long totalIndexEntries = 0; Map<KeyExtent,MLong> counts = new TreeMap<KeyExtent,MLong>(); for (KeyExtent keyExtent : extents) counts.put(keyExtent, new MLong(0)); - + Text row = new Text(); FileSystem ns = fs.getVolumeByPath(mapFile).getFileSystem(); FileSKVIterator index = FileOperations.getInstance().openIndex(mapFile.toString(), ns, ns.getConf(), acuConf); - + try { while (index.hasTop()) { Key key = index.getTopKey(); totalIndexEntries++; key.getRow(row); - + for (Entry<KeyExtent,MLong> entry : counts.entrySet()) if (entry.getKey().contains(row)) entry.getValue().l++; - + index.next(); } } finally { @@ -544,7 +544,7 @@ public class FileUtil { log.error(e, e); } } - + Map<KeyExtent,Long> results = new TreeMap<KeyExtent,Long>(); for (KeyExtent keyExtent : extents) { double numEntries = counts.get(keyExtent).l; @@ -555,7 +555,7 @@ public class FileUtil { } return results; } - + public static Collection<String> toPathStrings(Collection<FileRef> refs) { ArrayList<String> ret = new ArrayList<String>(); for (FileRef fileRef : refs) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/Halt.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Halt.java b/server/base/src/main/java/org/apache/accumulo/server/util/Halt.java index 82eb639..aa45bff 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/Halt.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/Halt.java @@ -22,7 +22,7 @@ import org.apache.log4j.Logger; public class Halt { static private final Logger log = Logger.getLogger(Halt.class); - + public static void halt(final String msg) { halt(0, new Runnable() { public void run() { @@ -30,7 +30,7 @@ public class Halt { } }); } - + public static void halt(final String msg, int status) { halt(status, new Runnable() { public void run() { @@ -38,7 +38,7 @@ public class Halt { } }); } - + public static void halt(final int status, Runnable runnable) { try { // give ourselves a little time to try and do something @@ -48,7 +48,7 @@ public class Halt { Runtime.getRuntime().halt(status); } }.start(); - + if (runnable != null) runnable.run(); Runtime.getRuntime().halt(status); @@ -57,5 +57,5 @@ public class Halt { Runtime.getRuntime().halt(-1); } } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java index 8550d0b..c8b8dff 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java @@ -38,39 +38,40 @@ import org.apache.log4j.Logger; import com.beust.jcommander.Parameter; public class ListInstances { - + private static final Logger log = Logger.getLogger(ListInstances.class); - + private static final int NAME_WIDTH = 20; private static final int UUID_WIDTH = 37; private static final int MASTER_WIDTH = 30; - + private static final int ZOOKEEPER_TIMER_MILLIS = 30 * 1000; - + static class Opts extends Help { - @Parameter(names="--print-errors", description="display errors while listing instances") + @Parameter(names = "--print-errors", description = "display errors while listing instances") boolean printErrors = false; - @Parameter(names="--print-all", description="print information for all instances, not just those with names") + @Parameter(names = "--print-all", description = "print information for all instances, not just those with names") boolean printAll = false; - @Parameter(names={"-z", "--zookeepers"}, description="the zookeepers to contact") + @Parameter(names = {"-z", "--zookeepers"}, description = "the zookeepers to contact") String keepers = null; } + static Opts opts = new Opts(); static int errors = 0; - + public static void main(String[] args) { opts.parseArgs(ListInstances.class.getName(), args); - + if (opts.keepers == null) { opts.keepers = SiteConfiguration.getInstance().get(Property.INSTANCE_ZK_HOST); } - + String keepers = opts.keepers; boolean printAll = opts.printAll; boolean printErrors = opts.printErrors; - + listInstances(keepers, printAll, printErrors); - + } static synchronized void listInstances(String keepers, boolean printAll, boolean printErrors) { @@ -81,17 +82,17 @@ public class ListInstances { ZooCache cache = new ZooCache(keepers, ZOOKEEPER_TIMER_MILLIS); TreeMap<String,UUID> instanceNames = getInstanceNames(rdr, printErrors); - + System.out.println(); printHeader(); - + for (Entry<String,UUID> entry : instanceNames.entrySet()) { printInstanceInfo(cache, entry.getKey(), entry.getValue(), printErrors); } - + TreeSet<UUID> instancedIds = getInstanceIDs(rdr, printErrors); instancedIds.removeAll(instanceNames.values()); - + if (printAll) { for (UUID uuid : instancedIds) { printInstanceInfo(cache, null, uuid, printErrors); @@ -102,57 +103,57 @@ public class ListInstances { } else { System.out.println(); } - + if (!printErrors && errors > 0) { System.err.println("WARN : There were " + errors + " errors, run with --print-errors to see more info"); } } - + private static class CharFiller implements Formattable { - + char c; - + CharFiller(char c) { this.c = c; } - + @Override public void formatTo(Formatter formatter, int flags, int width, int precision) { - + StringBuilder sb = new StringBuilder(); for (int i = 0; i < width; i++) sb.append(c); formatter.format(sb.toString()); } - + } - + private static void printHeader() { System.out.printf(" %-" + NAME_WIDTH + "s| %-" + UUID_WIDTH + "s| %-" + MASTER_WIDTH + "s%n", "Instance Name", "Instance ID", "Master"); System.out.printf("%" + (NAME_WIDTH + 1) + "s+%" + (UUID_WIDTH + 1) + "s+%" + (MASTER_WIDTH + 1) + "s%n", new CharFiller('-'), new CharFiller('-'), new CharFiller('-')); - + } - + private static void printInstanceInfo(ZooCache cache, String instanceName, UUID iid, boolean printErrors) { String master = getMaster(cache, iid, printErrors); if (instanceName == null) { instanceName = ""; } - + if (master == null) { master = ""; } - + System.out.printf("%" + NAME_WIDTH + "s |%" + UUID_WIDTH + "s |%" + MASTER_WIDTH + "s%n", "\"" + instanceName + "\"", iid, master); } - + private static String getMaster(ZooCache cache, UUID iid, boolean printErrors) { - + if (iid == null) { return null; } - + try { String masterLocPath = Constants.ZROOT + "/" + iid + Constants.ZMASTER_LOCK; byte[] master = ZooLock.getLockData(cache, masterLocPath, null); @@ -165,22 +166,22 @@ public class ListInstances { return null; } } - + private static TreeMap<String,UUID> getInstanceNames(ZooReader zk, boolean printErrors) { - + String instancesPath = Constants.ZROOT + Constants.ZINSTANCES; - + TreeMap<String,UUID> tm = new TreeMap<String,UUID>(); - + List<String> names; - + try { names = zk.getChildren(instancesPath); } catch (Exception e) { handleException(e, printErrors); return tm; } - + for (String name : names) { String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name; try { @@ -191,16 +192,16 @@ public class ListInstances { tm.put(name, null); } } - + return tm; } - + private static TreeSet<UUID> getInstanceIDs(ZooReader zk, boolean printErrors) { TreeSet<UUID> ts = new TreeSet<UUID>(); - + try { List<String> children = zk.getChildren(Constants.ZROOT); - + for (String iid : children) { if (iid.equals("instances")) continue; @@ -213,15 +214,15 @@ public class ListInstances { } catch (Exception e) { handleException(e, printErrors); } - + return ts; } - + private static void handleException(Exception e, boolean printErrors) { if (printErrors) { log.error(e); } - + errors++; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java b/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java index b288aac..e90d1dd 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java @@ -37,7 +37,7 @@ import org.apache.accumulo.server.fs.VolumeManager.FileType; import org.apache.hadoop.fs.Path; /** - * + * */ public class ListVolumesUsed { http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java index 35bb8f5..5d49fa7 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java @@ -41,22 +41,22 @@ import org.apache.hadoop.fs.Path; import com.google.common.net.HostAndPort; public class LocalityCheck { - + public int run(String[] args) throws Exception { ClientOpts opts = new ClientOpts(); opts.parseArgs(LocalityCheck.class.getName(), args); - + VolumeManager fs = VolumeManagerImpl.get(); Connector connector = opts.getConnector(); Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); scanner.fetchColumnFamily(DataFileColumnFamily.NAME); scanner.setRange(MetadataSchema.TabletsSection.getRange()); - + Map<String,Long> totalBlocks = new HashMap<String,Long>(); Map<String,Long> localBlocks = new HashMap<String,Long>(); ArrayList<String> files = new ArrayList<String>(); - + for (Entry<Key,Value> entry : scanner) { Key key = entry.getKey(); if (key.compareColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) { @@ -66,7 +66,7 @@ public class LocalityCheck { addBlocks(fs, host, files, totalBlocks, localBlocks); files.clear(); } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) { - + files.add(fs.getFullPath(key).toString()); } } @@ -78,7 +78,7 @@ public class LocalityCheck { } return 0; } - + private void addBlocks(VolumeManager fs, String host, ArrayList<String> files, Map<String,Long> totalBlocks, Map<String,Long> localBlocks) throws Exception { long allBlocks = 0; long matchingBlocks = 0; @@ -105,7 +105,7 @@ public class LocalityCheck { totalBlocks.put(host, allBlocks + totalBlocks.get(host)); localBlocks.put(host, matchingBlocks + localBlocks.get(host)); } - + public static void main(String[] args) throws Exception { LocalityCheck check = new LocalityCheck(); System.exit(check.run(args)); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java b/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java index 241eef3..be5a7c8 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java @@ -30,29 +30,29 @@ import org.apache.accumulo.server.security.handler.Authenticator; import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; /** - * + * */ public class LoginProperties { - + public static void main(String[] args) throws Exception { AccumuloConfiguration config = new ServerConfigurationFactory(HdfsZooInstance.getInstance()).getConfiguration(); Authenticator authenticator = AccumuloVFSClassLoader.getClassLoader().loadClass(config.get(Property.INSTANCE_SECURITY_AUTHENTICATOR)) .asSubclass(Authenticator.class).newInstance(); - + List<Set<TokenProperty>> tokenProps = new ArrayList<Set<TokenProperty>>(); - + for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) { tokenProps.add(tokenType.newInstance().getProperties()); } - + System.out.println("Supported token types for " + authenticator.getClass().getName() + " are : "); for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) { System.out.println("\t" + tokenType.getName() + ", which accepts the following properties : "); - + for (TokenProperty tokenProperty : tokenType.newInstance().getProperties()) { System.out.println("\t\t" + tokenProperty); } - + System.out.println(); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java index 88316c6..80a6734 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java @@ -60,94 +60,94 @@ import org.apache.log4j.Logger; import org.apache.zookeeper.KeeperException; /** - * + * */ public class MasterMetadataUtil { - + private static final Logger log = Logger.getLogger(MasterMetadataUtil.class); - - public static void addNewTablet(ClientContext context, KeyExtent extent, String path, TServerInstance location, - Map<FileRef,DataFileValue> datafileSizes, Map<FileRef,Long> bulkLoadedFiles, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) { + + public static void addNewTablet(ClientContext context, KeyExtent extent, String path, TServerInstance location, Map<FileRef,DataFileValue> datafileSizes, + Map<FileRef,Long> bulkLoadedFiles, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) { Mutation m = extent.getPrevRowUpdateMutation(); - + TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8))); TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8))); if (lastFlushID > 0) TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value(("" + lastFlushID).getBytes())); if (lastCompactID > 0) TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value(("" + lastCompactID).getBytes())); - + if (location != null) { location.putLocation(m); location.clearFutureLocation(m); } - + for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) { m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode())); } - + for (Entry<FileRef,Long> entry : bulkLoadedFiles.entrySet()) { byte[] tidBytes = Long.toString(entry.getValue()).getBytes(); m.put(TabletsSection.BulkFileColumnFamily.NAME, entry.getKey().meta(), new Value(tidBytes)); } - + MetadataTableUtil.update(context, zooLock, m, extent); } - + public static KeyExtent fixSplit(ClientContext context, Text metadataEntry, SortedMap<ColumnFQ,Value> columns, TServerInstance tserver, ZooLock lock) throws AccumuloException, IOException { log.info("Incomplete split " + metadataEntry + " attempting to fix"); - + Value oper = columns.get(TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN); - + if (columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN) == null) { throw new IllegalArgumentException("Metadata entry does not have split ratio (" + metadataEntry + ")"); } - + double splitRatio = Double.parseDouble(new String(columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN).get(), UTF_8)); - + Value prevEndRowIBW = columns.get(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN); - + if (prevEndRowIBW == null) { throw new IllegalArgumentException("Metadata entry does not have prev row (" + metadataEntry + ")"); } - + Value time = columns.get(TabletsSection.ServerColumnFamily.TIME_COLUMN); - + if (time == null) { throw new IllegalArgumentException("Metadata entry does not have time (" + metadataEntry + ")"); } - + Value flushID = columns.get(TabletsSection.ServerColumnFamily.FLUSH_COLUMN); long initFlushID = -1; if (flushID != null) initFlushID = Long.parseLong(flushID.toString()); - + Value compactID = columns.get(TabletsSection.ServerColumnFamily.COMPACT_COLUMN); long initCompactID = -1; if (compactID != null) initCompactID = Long.parseLong(compactID.toString()); - + Text metadataPrevEndRow = KeyExtent.decodePrevEndRow(prevEndRowIBW); - + Text table = (new KeyExtent(metadataEntry, (Text) null)).getTableId(); - + return fixSplit(context, table, metadataEntry, metadataPrevEndRow, oper, splitRatio, tserver, time.toString(), initFlushID, initCompactID, lock); } - + private static KeyExtent fixSplit(ClientContext context, Text table, Text metadataEntry, Text metadataPrevEndRow, Value oper, double splitRatio, TServerInstance tserver, String time, long initFlushID, long initCompactID, ZooLock lock) throws AccumuloException, IOException { if (metadataPrevEndRow == null) // something is wrong, this should not happen... if a tablet is split, it will always have a // prev end row.... throw new AccumuloException("Split tablet does not have prev end row, something is amiss, extent = " + metadataEntry); - + // check to see if prev tablet exist in metadata tablet Key prevRowKey = new Key(new Text(KeyExtent.getMetadataEntry(table, metadataPrevEndRow))); - + ScannerImpl scanner2 = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY); scanner2.setRange(new Range(prevRowKey, prevRowKey.followingKey(PartialKey.ROW))); - + VolumeManager fs = VolumeManagerImpl.get(); if (!scanner2.iterator().hasNext()) { log.info("Rolling back incomplete split " + metadataEntry + " " + metadataPrevEndRow); @@ -155,34 +155,34 @@ public class MasterMetadataUtil { return new KeyExtent(metadataEntry, KeyExtent.decodePrevEndRow(oper)); } else { log.info("Finishing incomplete split " + metadataEntry + " " + metadataPrevEndRow); - + List<FileRef> highDatafilesToRemove = new ArrayList<FileRef>(); - + Scanner scanner3 = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY); Key rowKey = new Key(metadataEntry); - + SortedMap<FileRef,DataFileValue> origDatafileSizes = new TreeMap<FileRef,DataFileValue>(); SortedMap<FileRef,DataFileValue> highDatafileSizes = new TreeMap<FileRef,DataFileValue>(); SortedMap<FileRef,DataFileValue> lowDatafileSizes = new TreeMap<FileRef,DataFileValue>(); scanner3.fetchColumnFamily(DataFileColumnFamily.NAME); scanner3.setRange(new Range(rowKey, rowKey.followingKey(PartialKey.ROW))); - + for (Entry<Key,Value> entry : scanner3) { if (entry.getKey().compareColumnFamily(DataFileColumnFamily.NAME) == 0) { origDatafileSizes.put(new FileRef(fs, entry.getKey()), new DataFileValue(entry.getValue().get())); } } - + MetadataTableUtil.splitDatafiles(table, metadataPrevEndRow, splitRatio, new HashMap<FileRef,FileUtil.FileInfo>(), origDatafileSizes, lowDatafileSizes, highDatafileSizes, highDatafilesToRemove); - + MetadataTableUtil.finishSplit(metadataEntry, highDatafileSizes, highDatafilesToRemove, context, lock); - + return new KeyExtent(metadataEntry, KeyExtent.encodePrevEndRow(metadataPrevEndRow)); } - + } - + private static TServerInstance getTServerInstance(String address, ZooLock zooLock) { while (true) { try { @@ -195,51 +195,51 @@ public class MasterMetadataUtil { UtilWaitThread.sleep(1000); } } - + public static void replaceDatafiles(ClientContext context, KeyExtent extent, Set<FileRef> datafilesToDelete, Set<FileRef> scanFiles, FileRef path, Long compactionId, DataFileValue size, String address, TServerInstance lastLocation, ZooLock zooLock) throws IOException { replaceDatafiles(context, extent, datafilesToDelete, scanFiles, path, compactionId, size, address, lastLocation, zooLock, true); } - + public static void replaceDatafiles(ClientContext context, KeyExtent extent, Set<FileRef> datafilesToDelete, Set<FileRef> scanFiles, FileRef path, Long compactionId, DataFileValue size, String address, TServerInstance lastLocation, ZooLock zooLock, boolean insertDeleteFlags) throws IOException { - + if (insertDeleteFlags) { // add delete flags for those paths before the data file reference is removed MetadataTableUtil.addDeleteEntries(extent, datafilesToDelete, context); } - + // replace data file references to old mapfiles with the new mapfiles Mutation m = new Mutation(extent.getMetadataEntry()); - + for (FileRef pathToRemove : datafilesToDelete) m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta()); - + for (FileRef scanFile : scanFiles) m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value(new byte[0])); - + if (size.getNumEntries() > 0) m.put(DataFileColumnFamily.NAME, path.meta(), new Value(size.encode())); - + if (compactionId != null) TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value(("" + compactionId).getBytes())); - + TServerInstance self = getTServerInstance(address, zooLock); self.putLastLocation(m); - + // remove the old location if (lastLocation != null && !lastLocation.equals(self)) lastLocation.clearLastLocation(m); - + MetadataTableUtil.update(context, zooLock, m, extent); } - + /** * new data file update function adds one data file to a tablet's list - * + * * @param path * should be relative to the table directory - * + * */ public static void updateTabletDataFile(ClientContext context, KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time, Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) { @@ -292,6 +292,7 @@ public class MasterMetadataUtil { /** * Create an update that updates a tablet + * * @return A Mutation to update a tablet from the given information */ protected static Mutation getUpdateForTabletDataFile(KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time, http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index 524abb0..ed7626e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -182,8 +182,7 @@ public class MetadataTableUtil { } } - public static void updateTabletDataFile(long tid, KeyExtent extent, Map<FileRef,DataFileValue> estSizes, String time, ClientContext context, - ZooLock zooLock) { + public static void updateTabletDataFile(long tid, KeyExtent extent, Map<FileRef,DataFileValue> estSizes, String time, ClientContext context, ZooLock zooLock) { Mutation m = new Mutation(extent.getMetadataEntry()); byte[] tidBytes = Long.toString(tid).getBytes(UTF_8); @@ -506,8 +505,8 @@ public class MetadataTableUtil { } } - public static Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> getFileAndLogEntries(ClientContext context, KeyExtent extent) - throws KeeperException, InterruptedException, IOException { + public static Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> getFileAndLogEntries(ClientContext context, KeyExtent extent) throws KeeperException, + InterruptedException, IOException { ArrayList<LogEntry> result = new ArrayList<LogEntry>(); TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>(); @@ -891,8 +890,8 @@ public class MetadataTableUtil { Key k = entry.getKey(); Mutation m = new Mutation(k.getRow()); m.putDelete(k.getColumnFamily(), k.getColumnQualifier()); - String dir = volumeManager.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR - + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES)); + String dir = volumeManager.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId + + Path.SEPARATOR + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES)); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(UTF_8))); bw.addMutation(m); @@ -983,8 +982,8 @@ public class MetadataTableUtil { * During an upgrade from 1.6 to 1.7, we need to add the replication table */ public static void createReplicationTable(ClientContext context) throws IOException { - String dir = VolumeManagerImpl.get().choose(Optional.of(ReplicationTable.ID), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + ReplicationTable.ID - + Constants.DEFAULT_TABLET_LOCATION; + String dir = VolumeManagerImpl.get().choose(Optional.of(ReplicationTable.ID), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + + ReplicationTable.ID + Constants.DEFAULT_TABLET_LOCATION; Mutation m = new Mutation(new Text(KeyExtent.getMetadataEntry(new Text(ReplicationTable.ID), null))); m.put(DIRECTORY_COLUMN.getColumnFamily(), DIRECTORY_COLUMN.getColumnQualifier(), 0, new Value(dir.getBytes(UTF_8))); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java b/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java index 0666297..b3c198e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java @@ -31,26 +31,26 @@ import org.apache.log4j.Logger; import com.beust.jcommander.Parameter; public class RandomWriter { - + private static String table_name = "test_write_table"; private static int num_columns_per_row = 1; private static int num_payload_bytes = 1024; private static final Logger log = Logger.getLogger(RandomWriter.class); - + public static class RandomMutationGenerator implements Iterable<Mutation>, Iterator<Mutation> { private long max_mutations; private int mutations_so_far = 0; private Random r = new Random(); private static final Logger log = Logger.getLogger(RandomMutationGenerator.class); - + public RandomMutationGenerator(long num_mutations) { max_mutations = num_mutations; } - + public boolean hasNext() { return mutations_so_far < max_mutations; } - + public Mutation next() { Text row_value = new Text(Long.toString(((r.nextLong() & 0x7fffffffffffffffl) / 177) % 100000000000l)); Mutation m = new Mutation(row_value); @@ -66,27 +66,31 @@ public class RandomWriter { } return m; } - + public void remove() { mutations_so_far++; } - + @Override public Iterator<Mutation> iterator() { return this; } } + static class Opts extends ClientOnDefaultTable { - @Parameter(names="--count", description="number of mutations to write", required=true) + @Parameter(names = "--count", description = "number of mutations to write", required = true) long count; - - Opts(String table) { super(table); } + + Opts(String table) { + super(table); + } } + public static void main(String[] args) throws Exception { Opts opts = new Opts(table_name); BatchWriterOpts bwOpts = new BatchWriterOpts(); opts.parseArgs(RandomWriter.class.getName(), args, bwOpts); - + long start = System.currentTimeMillis(); log.info("starting at " + start + " for user " + opts.principal); try { @@ -100,9 +104,9 @@ public class RandomWriter { throw e; } long stop = System.currentTimeMillis(); - + log.info("stopping at " + stop); log.info("elapsed: " + (((double) stop - (double) start) / 1000.0)); } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java index de360fe..77ffca3 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java @@ -112,7 +112,8 @@ public class RandomizeVolumes { Key key = entry.getKey(); Mutation m = new Mutation(key.getRow()); - final String newLocation = vm.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory; + final String newLocation = vm.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + + tableId + Path.SEPARATOR + directory; m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8))); if (log.isTraceEnabled()) { log.trace("Replacing " + oldLocation + " with " + newLocation); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java b/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java index d5b586c..b5cf510 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java @@ -57,7 +57,7 @@ import com.beust.jcommander.Parameter; /** * Remove file entries for map files that don't exist. - * + * */ public class RemoveEntriesForMissingFiles { http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java index b08bf90..e5a2add 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java @@ -40,17 +40,17 @@ import org.xml.sax.helpers.DefaultHandler; import com.beust.jcommander.Parameter; public class RestoreZookeeper { - + private static class Restore extends DefaultHandler { IZooReaderWriter zk = null; Stack<String> cwd = new Stack<String>(); boolean overwrite = false; - + Restore(IZooReaderWriter zk, boolean overwrite) { this.zk = zk; this.overwrite = overwrite; } - + @Override public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException { if ("node".equals(name)) { @@ -73,12 +73,12 @@ public class RestoreZookeeper { create(root, "", UTF_8.name()); } } - + @Override public void endElement(String uri, String localName, String name) throws SAXException { cwd.pop(); } - + // assume UTF-8 if not "base64" private void create(String path, String value, String encoding) { byte[] data = value.getBytes(UTF_8); @@ -97,7 +97,7 @@ public class RestoreZookeeper { } } } - + static class Opts extends Help { @Parameter(names = {"-z", "--keepers"}) String keepers = "localhost:2181"; @@ -106,17 +106,17 @@ public class RestoreZookeeper { @Parameter(names = "--file") String file; } - + public static void main(String[] args) throws Exception { Logger.getRootLogger().setLevel(Level.WARN); Opts opts = new Opts(); opts.parseArgs(RestoreZookeeper.class.getName(), args); - + InputStream in = System.in; if (opts.file != null) { in = new FileInputStream(opts.file); } - + SAXParserFactory factory = SAXParserFactory.newInstance(); SAXParser parser = factory.newSAXParser(); parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite)); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java index 63f1343..2c192cf 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java @@ -254,7 +254,7 @@ public class SendLogToChainsaw extends XMLLayout { } /** - * + * * @param args * <ol> * <li>path to log directory</li> http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java index a376ed6..49a6971 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java @@ -38,14 +38,14 @@ public class SystemPropUtil { log.warn("Ignoring property {} it is null, an invalid format, or not capable of being changed in zookeeper", property); return false; } - + // create the zk node for this property and set it's data to the specified value String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property; ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE); - + return true; } - + public static void removeSystemProperty(String property) throws InterruptedException, KeeperException { String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property; ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.FAIL); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/TableInfoUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableInfoUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableInfoUtil.java index cc91ef3..6aa937f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TableInfoUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableInfoUtil.java @@ -25,10 +25,10 @@ import org.apache.accumulo.core.master.thrift.TableInfo; import org.apache.accumulo.core.master.thrift.TabletServerStatus; /** - * + * */ public class TableInfoUtil { - + public static void add(TableInfo total, TableInfo more) { if (total.minors == null) total.minors = new Compacting(); @@ -58,7 +58,7 @@ public class TableInfoUtil { total.queryByteRate += more.queryByteRate; total.scanRate += more.scanRate; } - + public static TableInfo summarizeTableStats(TabletServerStatus status) { TableInfo summary = new TableInfo(); summary.majors = new Compacting(); @@ -69,7 +69,7 @@ public class TableInfoUtil { } return summary; } - + public static Map<String,Double> summarizeTableStats(MasterMonitorInfo mmi) { Map<String,Double> compactingByTable = new HashMap<String,Double>(); if (mmi != null && mmi.tServerInfo != null) { @@ -84,5 +84,5 @@ public class TableInfoUtil { } return compactingByTable; } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java index 5e6542d..ab14311 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java @@ -31,31 +31,31 @@ public class TablePropUtil { public static boolean setTableProperty(String tableId, String property, String value) throws KeeperException, InterruptedException { if (!isPropertyValid(property, value)) return false; - + // create the zk node for per-table properties for this table if it doesn't already exist String zkTablePath = getTablePath(tableId); ZooReaderWriter.getInstance().putPersistentData(zkTablePath, new byte[0], NodeExistsPolicy.SKIP); - + // create the zk node for this property and set it's data to the specified value String zPath = zkTablePath + "/" + property; ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE); - + return true; } - + public static boolean isPropertyValid(String property, String value) { Property p = Property.getPropertyByKey(property); if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property)) return false; - + return true; } - + public static void removeTableProperty(String tableId, String property) throws InterruptedException, KeeperException { String zPath = getTablePath(tableId) + "/" + property; ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP); } - + private static String getTablePath(String tablename) { return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tablename + Constants.ZTABLE_CONF; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/TabletIterator.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletIterator.java b/server/base/src/main/java/org/apache/accumulo/server/util/TabletIterator.java index 8dd414b..3bc6c96 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletIterator.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletIterator.java @@ -40,56 +40,52 @@ import org.apache.log4j.Logger; * This class iterates over the metadata table returning all key values for a tablet in one chunk. As it scans the metadata table it checks the correctness of * the metadata table, and rescans if needed. So the tablet key/values returned by this iterator should satisfy the sorted linked list property of the metadata * table. - * + * * The purpose of this is to hide inconsistencies caused by splits and detect anomalies in the metadata table. - * + * * If a tablet that was returned by this iterator is subsequently deleted from the metadata table, then this iterator will throw a TabletDeletedException. This * could occur when a table is merged. - * - * + * + * */ public class TabletIterator implements Iterator<Map<Key,Value>> { - + private static final Logger log = Logger.getLogger(TabletIterator.class); - + private SortedMap<Key,Value> currentTabletKeys; - + private Text lastTablet; - + private Scanner scanner; private Iterator<Entry<Key,Value>> iter; - + private boolean returnPrevEndRow; - + private boolean returnDir; - + private Range range; - + public static class TabletDeletedException extends RuntimeException { - - /** - * - */ - + private static final long serialVersionUID = 1L; - + public TabletDeletedException(String msg) { super(msg); } } - + /* * public TabletIterator(String table, boolean returnPrevEndRow){ - * + * * } */ - + /** - * + * * @param s * A scanner over the entire metadata table configure to fetch needed columns. */ - + public TabletIterator(Scanner s, Range range, boolean returnPrevEndRow, boolean returnDir) { this.scanner = s; this.range = range; @@ -100,129 +96,129 @@ public class TabletIterator implements Iterator<Map<Key,Value>> { this.returnPrevEndRow = returnPrevEndRow; this.returnDir = returnDir; } - + @Override public boolean hasNext() { while (currentTabletKeys == null) { - + currentTabletKeys = scanToPrevEndRow(); if (currentTabletKeys.size() == 0) { break; } - + Key prevEndRowKey = currentTabletKeys.lastKey(); Value prevEndRowValue = currentTabletKeys.get(prevEndRowKey); - + if (!TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(prevEndRowKey)) { log.debug(currentTabletKeys); throw new RuntimeException("Unexpected key " + prevEndRowKey); } - + Text per = KeyExtent.decodePrevEndRow(prevEndRowValue); Text lastEndRow; - + if (lastTablet == null) { lastEndRow = null; } else { lastEndRow = new KeyExtent(lastTablet, (Text) null).getEndRow(); - + // do table transition sanity check String lastTable = new KeyExtent(lastTablet, (Text) null).getTableId().toString(); String currentTable = new KeyExtent(prevEndRowKey.getRow(), (Text) null).getTableId().toString(); - + if (!lastTable.equals(currentTable) && (per != null || lastEndRow != null)) { log.info("Metadata inconsistency on table transition : " + lastTable + " " + currentTable + " " + per + " " + lastEndRow); - + currentTabletKeys = null; resetScanner(); - + UtilWaitThread.sleep(250); - + continue; } } - + boolean perEqual = (per == null && lastEndRow == null) || (per != null && lastEndRow != null && per.equals(lastEndRow)); - + if (!perEqual) { - + log.info("Metadata inconsistency : " + per + " != " + lastEndRow + " metadataKey = " + prevEndRowKey); - + currentTabletKeys = null; resetScanner(); - + UtilWaitThread.sleep(250); - + continue; - + } // this tablet is good, so set it as the last tablet lastTablet = prevEndRowKey.getRow(); } - + return currentTabletKeys.size() > 0; } - + @Override public Map<Key,Value> next() { - + if (!hasNext()) throw new NoSuchElementException(); - + Map<Key,Value> tmp = currentTabletKeys; currentTabletKeys = null; - + Set<Entry<Key,Value>> es = tmp.entrySet(); Iterator<Entry<Key,Value>> esIter = es.iterator(); - + while (esIter.hasNext()) { Map.Entry<Key,Value> entry = esIter.next(); if (!returnPrevEndRow && TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(entry.getKey())) { esIter.remove(); } - + if (!returnDir && TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(entry.getKey())) { esIter.remove(); } } - + return tmp; } - + @Override public void remove() { throw new UnsupportedOperationException(); } - + private SortedMap<Key,Value> scanToPrevEndRow() { - + Text curMetaDataRow = null; - + TreeMap<Key,Value> tm = new TreeMap<Key,Value>(); - + boolean sawPrevEndRow = false; - + while (true) { while (iter.hasNext()) { Entry<Key,Value> entry = iter.next(); - + if (curMetaDataRow == null) { curMetaDataRow = entry.getKey().getRow(); } - + if (!curMetaDataRow.equals(entry.getKey().getRow())) { // tablet must not have a prev end row, try scanning again break; } - + tm.put(entry.getKey(), entry.getValue()); - + if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(entry.getKey())) { sawPrevEndRow = true; break; } } - + if (!sawPrevEndRow && tm.size() > 0) { log.warn("Metadata problem : tablet " + curMetaDataRow + " has no prev end row"); resetScanner(); @@ -233,14 +229,14 @@ public class TabletIterator implements Iterator<Map<Key,Value>> { break; } } - + return tm; } - + protected void resetScanner() { - + Range range; - + if (lastTablet == null) { range = this.range; } else { @@ -252,19 +248,19 @@ public class TabletIterator implements Iterator<Map<Key,Value>> { Entry<Key,Value> entry : scanner) { count++; } - + if (count == 0) throw new TabletDeletedException("Tablet " + lastTablet + " was deleted while iterating"); - + // start right after the last good tablet range = new Range(new Key(lastTablet).followingKey(PartialKey.ROW), true, this.range.getEndKey(), this.range.isEndKeyInclusive()); } - + log.info("Resetting " + MetadataTable.NAME + " scanner to " + range); - + scanner.setRange(range); iter = scanner.iterator(); - + } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java index c0e1a9b..b1e2d35 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java @@ -24,8 +24,8 @@ import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.fs.VolumeManagerImpl; -import org.apache.hadoop.fs.FileSystem; import org.apache.accumulo.server.tablets.UniqueNameAllocator; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; @@ -33,15 +33,15 @@ import org.apache.log4j.Logger; import com.google.common.base.Optional; public class TabletOperations { - + private static final Logger log = Logger.getLogger(TabletOperations.class); - + public static String createTabletDirectory(VolumeManager fs, String tableId, Text endRow) { String lowDirectory; - + UniqueNameAllocator namer = UniqueNameAllocator.getInstance(); String volume = fs.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR; - + while (true) { try { if (endRow == null) { @@ -54,7 +54,7 @@ public class TabletOperations { log.warn("Failed to create " + lowDirectoryPath + " for unknown reason"); } else { lowDirectory = "/" + Constants.GENERATED_TABLET_DIRECTORY_PREFIX + namer.getNextName(); - Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory); + Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory); if (fs.exists(lowDirectoryPath)) throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath); if (fs.mkdirs(lowDirectoryPath)) { @@ -65,13 +65,13 @@ public class TabletOperations { } catch (IOException e) { log.warn(e); } - + log.warn("Failed to create dir for tablet in table " + tableId + " in volume " + volume + " + will retry ..."); UtilWaitThread.sleep(3000); - + } } - + public static String createTabletDirectory(String tableDir, Text endRow) { while (true) { try { http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java index 307bb0c..01bb926 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java @@ -33,34 +33,35 @@ import org.apache.accumulo.server.zookeeper.ZooReaderWriter; import com.beust.jcommander.Parameter; public class TabletServerLocks { - + static class Opts extends Help { - @Parameter(names="-list") + @Parameter(names = "-list") boolean list = false; - @Parameter(names="-delete") + @Parameter(names = "-delete") String delete = null; } + public static void main(String[] args) throws Exception { - + Instance instance = HdfsZooInstance.getInstance(); String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS; Opts opts = new Opts(); opts.parseArgs(TabletServerLocks.class.getName(), args); - + ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()); - + if (opts.list) { IZooReaderWriter zoo = ZooReaderWriter.getInstance(); - + List<String> tabletServers = zoo.getChildren(tserverPath); - + for (String tabletServer : tabletServers) { byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null); String holder = null; if (lockData != null) { holder = new String(lockData, UTF_8); } - + System.out.printf("%32s %16s%n", tabletServer, holder); } } else if (opts.delete != null) { @@ -68,7 +69,7 @@ public class TabletServerLocks { } else { System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>"); } - + } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java b/server/base/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java index 8e6b339..98de6be 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java @@ -61,36 +61,36 @@ import com.google.common.net.HostAndPort; public class VerifyTabletAssignments { private static final Logger log = Logger.getLogger(VerifyTabletAssignments.class); - + static class Opts extends ClientOpts { @Parameter(names = {"-v", "--verbose"}, description = "verbose mode (prints locations of tablets)") boolean verbose = false; } - + public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(VerifyTabletAssignments.class.getName(), args); - + ClientContext context = new ClientContext(opts.getInstance(), new Credentials(opts.principal, opts.getToken()), opts.getClientConfiguration()); Connector conn = opts.getConnector(); for (String table : conn.tableOperations().list()) checkTable(context, opts, table, null); - + } - + private static void checkTable(final ClientContext context, final Opts opts, String tableName, HashSet<KeyExtent> check) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, InterruptedException { - + if (check == null) System.out.println("Checking table " + tableName); else System.out.println("Checking table " + tableName + " again, failures " + check.size()); - + TreeMap<KeyExtent,String> tabletLocations = new TreeMap<KeyExtent,String>(); - + String tableId = Tables.getNameToIdMap(context.getInstance()).get(tableName); MetadataServicer.forTableId(context, tableId).getTabletLocations(tabletLocations); - + final HashSet<KeyExtent> failures = new HashSet<KeyExtent>(); Map<HostAndPort,List<KeyExtent>> extentsPerServer = new TreeMap<HostAndPort,List<KeyExtent>>(); @@ -102,7 +102,7 @@ public class VerifyTabletAssignments { System.out.println(" Tablet " + keyExtent + " has no location"); else if (opts.verbose) System.out.println(" Tablet " + keyExtent + " is located at " + loc); - + if (loc != null) { final HostAndPort parsedLoc = HostAndPort.fromString(loc); List<KeyExtent> extentList = extentsPerServer.get(parsedLoc); @@ -110,35 +110,35 @@ public class VerifyTabletAssignments { extentList = new ArrayList<KeyExtent>(); extentsPerServer.put(parsedLoc, extentList); } - + if (check == null || check.contains(keyExtent)) extentList.add(keyExtent); } } - + ExecutorService tp = Executors.newFixedThreadPool(20); for (final Entry<HostAndPort,List<KeyExtent>> entry : extentsPerServer.entrySet()) { Runnable r = new Runnable() { - + @Override public void run() { try { checkTabletServer(context, entry, failures); } catch (Exception e) { - log.error("Failure on tablet server '"+entry.getKey()+".", e); + log.error("Failure on tablet server '" + entry.getKey() + ".", e); failures.addAll(entry.getValue()); } } - + }; - + tp.execute(r); } - + tp.shutdown(); - + while (!tp.awaitTermination(1, TimeUnit.HOURS)) {} - + if (failures.size() > 0) checkTable(context, opts, tableName, failures); } @@ -154,32 +154,32 @@ public class VerifyTabletAssignments { private static void checkTabletServer(ClientContext context, Entry<HostAndPort,List<KeyExtent>> entry, HashSet<KeyExtent> failures) throws ThriftSecurityException, TException, NoSuchScanIDException { TabletClientService.Iface client = ThriftUtil.getTServerClient(entry.getKey(), context); - + Map<TKeyExtent,List<TRange>> batch = new TreeMap<TKeyExtent,List<TRange>>(); - + for (KeyExtent keyExtent : entry.getValue()) { Text row = keyExtent.getEndRow(); Text row2 = null; - + if (row == null) { row = keyExtent.getPrevEndRow(); - + if (row != null) { row = new Text(row); row.append(new byte[] {'a'}, 0, 1); } else { row = new Text("1234567890"); } - + row2 = new Text(row); row2.append(new byte[] {'!'}, 0, 1); } else { row = new Text(row); row2 = new Text(row); - + row.getBytes()[row.getLength() - 1] = (byte) (row.getBytes()[row.getLength() - 1] - 1); } - + Range r = new Range(row, true, row2, false); batch.put(keyExtent.toThrift(), Collections.singletonList(r.toThrift())); } @@ -192,15 +192,15 @@ public class VerifyTabletAssignments { if (is.result.more) { MultiScanResult result = client.continueMultiScan(tinfo, is.scanID); checkFailures(entry.getKey(), failures, result); - + while (result.more) { result = client.continueMultiScan(tinfo, is.scanID); checkFailures(entry.getKey(), failures, result); } } - + client.closeMultiScan(tinfo, is.scanID); - + ThriftUtil.returnClient((TServiceClient) client); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java b/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java index efcefde..0edcf71 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java @@ -27,16 +27,16 @@ import org.apache.hadoop.fs.Path; import com.beust.jcommander.Parameter; public class ZooKeeperMain { - + static class Opts extends Help { - + @Parameter(names = {"-z", "--keepers"}, description = "Comma separated list of zookeeper hosts (host:port,host:port)") String servers = null; - + @Parameter(names = {"-t", "--timeout"}, description = "timeout, in seconds to timeout the zookeeper connection") long timeout = 30; } - + public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(ZooKeeperMain.class.getName(), args); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java index 7fdbf13..ef182f1 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java @@ -20,67 +20,66 @@ import java.util.List; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; -import org.apache.accumulo.server.cli.ClientOpts; import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy; +import org.apache.accumulo.server.cli.ClientOpts; import org.apache.accumulo.server.zookeeper.ZooLock; import org.apache.accumulo.server.zookeeper.ZooReaderWriter; +import org.apache.log4j.Logger; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import org.apache.log4j.Logger; public class ZooZap { private static final Logger log = Logger.getLogger(ZooZap.class); - + static boolean verbose = false; - + private static void message(String msg) { if (verbose) System.out.println(msg); } - + static class Opts extends ClientOpts { - @Parameter(names="-master", description="remove master locks") + @Parameter(names = "-master", description = "remove master locks") boolean zapMaster = false; - @Parameter(names="-tservers", description="remove tablet server locks") + @Parameter(names = "-tservers", description = "remove tablet server locks") boolean zapTservers = false; - @Parameter(names="-tracers", description="remove tracer locks") + @Parameter(names = "-tracers", description = "remove tracer locks") boolean zapTracers = false; - @Parameter(names="-verbose", description="print out messages about progress") + @Parameter(names = "-verbose", description = "print out messages about progress") boolean verbose = false; String getTraceZKPath() { return super.getClientConfiguration().get(ClientProperty.TRACE_ZK_PATH); } } - + public static void main(String[] args) { Opts opts = new Opts(); opts.parseArgs(ZooZap.class.getName(), args); - - if (!opts.zapMaster && !opts.zapTservers && !opts.zapTracers) - { - new JCommander(opts).usage(); - return; + + if (!opts.zapMaster && !opts.zapTservers && !opts.zapTracers) { + new JCommander(opts).usage(); + return; } - + String iid = opts.getInstance().getInstanceID(); IZooReaderWriter zoo = ZooReaderWriter.getInstance(); - + if (opts.zapMaster) { String masterLockPath = Constants.ZROOT + "/" + iid + Constants.ZMASTER_LOCK; - + zapDirectory(zoo, masterLockPath); } - + if (opts.zapTservers) { String tserversPath = Constants.ZROOT + "/" + iid + Constants.ZTSERVERS; try { List<String> children = zoo.getChildren(tserversPath); for (String child : children) { message("Deleting " + tserversPath + "/" + child + " from zookeeper"); - + if (opts.zapMaster) ZooReaderWriter.getInstance().recursiveDelete(tserversPath + "/" + child, NodeMissingPolicy.SKIP); else { @@ -96,14 +95,14 @@ public class ZooZap { log.error(e); } } - + if (opts.zapTracers) { String path = opts.getTraceZKPath(); zapDirectory(zoo, path); } - + } - + private static void zapDirectory(IZooReaderWriter zoo, String path) { try { List<String> children = zoo.getChildren(path); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/time/BaseRelativeTime.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/time/BaseRelativeTime.java b/server/base/src/main/java/org/apache/accumulo/server/util/time/BaseRelativeTime.java index 393c6d2..73afc7e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/time/BaseRelativeTime.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/time/BaseRelativeTime.java @@ -18,25 +18,25 @@ package org.apache.accumulo.server.util.time; /** * Provide time from a local source and a hint from a time source. - * + * * RelativeTime and BaseRelativeTime are separated to provide unit tests of the core functionality of Relative timekeeping. - * + * */ public class BaseRelativeTime implements ProvidesTime { - + private long diff = 0; private long lastReportedTime = 0; ProvidesTime local; - + BaseRelativeTime(ProvidesTime real, long lastReportedTime) { this.local = real; this.lastReportedTime = lastReportedTime; } - + BaseRelativeTime(ProvidesTime real) { this(real, 0); } - + @Override synchronized public long currentTime() { long localNow = local.currentTime(); @@ -46,12 +46,12 @@ public class BaseRelativeTime implements ProvidesTime { lastReportedTime = result; return result; } - + synchronized public void updateTime(long advice) { long localNow = local.currentTime(); long diff = advice - localNow; // smooth in 20% of the change, not the whole thing. this.diff = (this.diff * 4 / 5) + diff / 5; } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/time/ProvidesTime.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/time/ProvidesTime.java b/server/base/src/main/java/org/apache/accumulo/server/util/time/ProvidesTime.java index 117fa5f..1042c32 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/time/ProvidesTime.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/time/ProvidesTime.java @@ -18,7 +18,7 @@ package org.apache.accumulo.server.util.time; /** * An interface for anything that returns the time in the same format as System.currentTimeMillis(). - * + * */ public interface ProvidesTime { long currentTime(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/server/base/src/main/java/org/apache/accumulo/server/util/time/RelativeTime.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/time/RelativeTime.java b/server/base/src/main/java/org/apache/accumulo/server/util/time/RelativeTime.java index 99581e9..bc48b10 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/time/RelativeTime.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/time/RelativeTime.java @@ -18,27 +18,27 @@ package org.apache.accumulo.server.util.time; /** * Provide time from System time and hints from another time source. - * + * * Provides a convenient static replacement for System.currentTimeMillis() */ public class RelativeTime extends BaseRelativeTime { - + private RelativeTime() { super(new SystemTime()); } - + private static BaseRelativeTime instance = new RelativeTime(); - + public static BaseRelativeTime getInstance() { return instance; } - + public static void setInstance(BaseRelativeTime newInstance) { instance = newInstance; } - + public static long currentTimeMillis() { return getInstance().currentTime(); } - + }
