Updated Branches: refs/heads/master fcc17fc0a -> 275d7941b
ACCUMULO-1961 Fix/remove some more trivial warnings Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7820120f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7820120f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7820120f Branch: refs/heads/master Commit: 7820120fb2ce4c18317ab77b6439197084ceac8f Parents: a72dd78 Author: Christopher Tubbs <ctubb...@apache.org> Authored: Wed Feb 5 15:36:27 2014 -0500 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Wed Feb 5 15:36:27 2014 -0500 ---------------------------------------------------------------------- .../core/client/ClientConfiguration.java | 2 +- .../lib/partition/RangePartitioner.java | 27 ++-- .../mapreduce/AccumuloInputFormatTest.java | 3 +- .../apache/accumulo/fate/util/AddressUtil.java | 27 ++-- .../minicluster/MiniAccumuloCluster.java | 2 - .../impl/MiniAccumuloConfigImpl.java | 2 - .../accumulo/server/util/SendLogToChainsaw.java | 69 +++++----- .../org/apache/accumulo/test/VerifyIngest.java | 89 ++++++------ .../test/continuous/UndefinedAnalyzer.java | 134 ++++++++++--------- .../accumulo/test/functional/TabletIT.java | 2 - 10 files changed, 177 insertions(+), 180 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/core/src/main/java/org/apache/accumulo/core/client/ClientConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientConfiguration.java b/core/src/main/java/org/apache/accumulo/core/client/ClientConfiguration.java index a3d51cf..b0ae537 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ClientConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ClientConfiguration.java @@ -128,7 +128,7 @@ public class ClientConfiguration extends CompositeConfiguration { * the one in the configuration. * * @see PropertiesConfiguration - * @see File.pathSeparator + * @see File#pathSeparator */ public static ClientConfiguration loadDefault() { return loadFromSearchPath(getDefaultSearchPath()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java index 3cb69ad..1569579 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java @@ -19,7 +19,6 @@ package org.apache.accumulo.core.client.mapreduce.lib.partition; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.URI; @@ -45,9 +44,9 @@ public class RangePartitioner extends Partitioner<Text,Writable> implements Conf private static final String PREFIX = RangePartitioner.class.getName(); private static final String CUTFILE_KEY = PREFIX + ".cutFile"; private static final String NUM_SUBBINS = PREFIX + ".subBins"; - + private Configuration conf; - + @Override public int getPartition(Text key, Writable value, int numPartitions) { try { @@ -56,21 +55,21 @@ public class RangePartitioner extends Partitioner<Text,Writable> implements Conf throw new RuntimeException(e); } } - + int findPartition(Text key, Text[] array, int numSubBins) { // find the bin for the range, and guarantee it is positive int index = Arrays.binarySearch(array, key); index = index < 0 ? (index + 1) * -1 : index; - + // both conditions work with numSubBins == 1, but this check is to avoid // hashing, when we don't need to, for speed if (numSubBins < 2) return index; return (key.toString().hashCode() & Integer.MAX_VALUE) % numSubBins + index * numSubBins; } - + private int _numSubBins = 0; - + private synchronized int getNumSubBins() { if (_numSubBins < 1) { // get number of sub-bins and guarantee it is positive @@ -78,14 +77,14 @@ public class RangePartitioner extends Partitioner<Text,Writable> implements Conf } return _numSubBins; } - + private Text cutPointArray[] = null; - + private synchronized Text[] getCutPoints() throws IOException { if (cutPointArray == null) { String cutFileName = conf.get(CUTFILE_KEY); Path[] cf = DistributedCacheHelper.getLocalCacheFiles(conf); - + if (cf != null) { for (Path path : cf) { if (path.toUri().getPath().endsWith(cutFileName.substring(cutFileName.lastIndexOf('/')))) { @@ -107,17 +106,17 @@ public class RangePartitioner extends Partitioner<Text,Writable> implements Conf } return cutPointArray; } - + @Override public Configuration getConf() { return conf; } - + @Override public void setConf(Configuration conf) { this.conf = conf; } - + /** * Sets the hdfs file name to use, containing a newline separated list of Base64 encoded split points that represent ranges for partitioning */ @@ -126,7 +125,7 @@ public class RangePartitioner extends Partitioner<Text,Writable> implements Conf DistributedCacheHelper.addCacheFile(uri, job.getConfiguration()); job.getConfiguration().set(CUTFILE_KEY, uri.getPath()); } - + /** * Sets the number of random sub-bins per range */ http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java index 165c7f7..54bd127 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java @@ -401,7 +401,8 @@ public class AccumuloInputFormatTest { @Test public void testEmptyColumnFamily() throws IOException { - Job job = Job.getInstance(); + @SuppressWarnings("deprecation") + Job job = new Job(); Set<Pair<Text,Text>> cols = new HashSet<Pair<Text,Text>>(); cols.add(new Pair<Text,Text>(new Text(""), null)); cols.add(new Pair<Text,Text>(new Text("foo"), new Text("bar"))); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/fate/src/main/java/org/apache/accumulo/fate/util/AddressUtil.java ---------------------------------------------------------------------- diff --git a/fate/src/main/java/org/apache/accumulo/fate/util/AddressUtil.java b/fate/src/main/java/org/apache/accumulo/fate/util/AddressUtil.java index 7a8c269..fce7a62 100644 --- a/fate/src/main/java/org/apache/accumulo/fate/util/AddressUtil.java +++ b/fate/src/main/java/org/apache/accumulo/fate/util/AddressUtil.java @@ -16,7 +16,7 @@ */ package org.apache.accumulo.fate.util; -import java.net.InetAddress; // workaround to enable @see/@link hyperlink +import java.net.InetAddress; import java.net.UnknownHostException; import java.security.Security; @@ -27,28 +27,29 @@ public class AddressUtil { private static final Logger log = Logger.getLogger(AddressUtil.class); /** - * Fetch the security value that determines how long DNS failures are cached. - * Looks up the security property 'networkaddress.cache.negative.ttl'. Should that fail returns - * the default value used in the Oracle JVM 1.4+, which is 10 seconds. - * - * @param originalException the host lookup that is the source of needing this lookup. maybe be null. + * Fetch the security value that determines how long DNS failures are cached. Looks up the security property 'networkaddress.cache.negative.ttl'. Should that + * fail returns the default value used in the Oracle JVM 1.4+, which is 10 seconds. + * + * @param originalException + * the host lookup that is the source of needing this lookup. maybe be null. * @return positive integer number of seconds - * @see java.net.InetAddress - * @throws IllegalArgumentException if dns failures are cached forever + * @see InetAddress + * @throws IllegalArgumentException + * if dns failures are cached forever */ static public int getAddressCacheNegativeTtl(UnknownHostException originalException) { int negativeTtl = 10; try { negativeTtl = Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl")); } catch (NumberFormatException exception) { - log.warn("Failed to get JVM negative DNS respones cache TTL due to format problem (e.g. this JVM might not have the " + - "property). Falling back to default based on Oracle JVM 1.6 (10s)", exception); + log.warn("Failed to get JVM negative DNS respones cache TTL due to format problem (e.g. this JVM might not have the " + + "property). Falling back to default based on Oracle JVM 1.6 (10s)", exception); } catch (SecurityException exception) { log.warn("Failed to get JVM negative DNS response cache TTL due to security manager. Falling back to default based on Oracle JVM 1.6 (10s)", exception); } if (-1 == negativeTtl) { - log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and host lookup failed. TTL can be changed with security property " + - "'networkaddress.cache.negative.ttl', see java.net.InetAddress.", originalException); + log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and host lookup failed. TTL can be changed with security property " + + "'networkaddress.cache.negative.ttl', see java.net.InetAddress.", originalException); throw new IllegalArgumentException(originalException); } else if (0 > negativeTtl) { log.warn("JVM specified negative DNS response cache TTL was negative (and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)"); @@ -56,5 +57,5 @@ public class AddressUtil { } return negativeTtl; } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java index 6b0b1ff..50bb14a 100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java @@ -42,7 +42,6 @@ public class MiniAccumuloCluster { impl = new MiniAccumuloClusterImpl(config); } - /** * * @param dir @@ -59,7 +58,6 @@ public class MiniAccumuloCluster { * @param config * initial configuration */ - @SuppressWarnings("deprecation") public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException { this(config.getImpl()); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java index 04d9f6e..2c1b326 100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java @@ -17,12 +17,10 @@ package org.apache.accumulo.minicluster.impl; import java.io.File; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.util.StringUtil; import org.apache.accumulo.minicluster.MemoryUnit; import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.server.util.PortUtils; http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java index 1f61b43..85f29ba 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java @@ -20,7 +20,6 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStreamReader; @@ -55,33 +54,33 @@ import com.beust.jcommander.IStringConverter; import com.beust.jcommander.Parameter; public class SendLogToChainsaw extends XMLLayout { - + private static Pattern logPattern = Pattern.compile( "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES); - + private File[] logFiles = null; - + private SocketFactory factory = SocketFactory.getDefault(); - + private WildcardFileFilter fileFilter = null; - + private Socket socket = null; - + private Pattern lineFilter = null; - + private LongRange dateFilter = null; - + private LevelRangeFilter levelFilter = null; - + public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception { - + // Set up the file name filter if (null != fileNameFilter) { fileFilter = new WildcardFileFilter(fileNameFilter); } else { fileFilter = new WildcardFileFilter("*"); } - + // Get the list of files that match File dir = new File(directory); if (dir.isDirectory()) { @@ -89,22 +88,22 @@ public class SendLogToChainsaw extends XMLLayout { } else { throw new IllegalArgumentException(directory + " is not a directory or is not readable."); } - + if (logFiles.length == 0) { throw new IllegalArgumentException("No files match the supplied filter."); } - + socket = factory.createSocket(host, port); - + lineFilter = Pattern.compile(regex); - + // Create Date Filter if (null != start) { if (end == null) end = new Date(System.currentTimeMillis()); dateFilter = new LongRange(start.getTime(), end.getTime()); } - + if (null != level) { Level base = Level.toLevel(level.toUpperCase()); levelFilter = new LevelRangeFilter(); @@ -113,7 +112,7 @@ public class SendLogToChainsaw extends XMLLayout { levelFilter.setLevelMax(Level.FATAL); } } - + public void processLogFiles() throws IOException { String line = null; String out = null; @@ -130,7 +129,7 @@ public class SendLogToChainsaw extends XMLLayout { throw e; } reader = new BufferedReader(isReader); - + try { line = reader.readLine(); while (null != line) { @@ -161,12 +160,12 @@ public class SendLogToChainsaw extends XMLLayout { } } } - + private String convertLine(String line, String threadName) throws UnsupportedEncodingException { String result = null; Matcher m = logPattern.matcher(line); if (m.matches()) { - + Calendar cal = Calendar.getInstance(); cal.setTime(new Date(System.currentTimeMillis())); Integer date = Integer.parseInt(m.group(1)); @@ -212,7 +211,7 @@ public class SendLogToChainsaw extends XMLLayout { } return result; } - + private static class DateConverter implements IStringConverter<Date> { @Override public Date convert(String value) { @@ -223,36 +222,36 @@ public class SendLogToChainsaw extends XMLLayout { throw new RuntimeException(e); } } - + } - + private static class Opts extends Help { - + @Parameter(names = {"-d", "--logDirectory"}, description = "ACCUMULO log directory path", required = true) String dir; - + @Parameter(names = {"-f", "--fileFilter"}, description = "filter to apply to names of logs") String filter; - + @Parameter(names = {"-h", "--host"}, description = "host where chainsaw is running", required = true) String hostname; - + @Parameter(names = {"-p", "--port"}, description = "port where XMLSocketReceiver is listening", required = true) int portnum; - + @Parameter(names = {"-s", "--start"}, description = "start date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class) Date startDate; - + @Parameter(names = {"-e", "--end"}, description = "end date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class) Date endDate; - + @Parameter(names = {"-l", "--level"}, description = "filter log level") String level; - + @Parameter(names = {"-m", "--messageFilter"}, description = "regex filter for log messages") String regex; } - + /** * * @param args @@ -270,9 +269,9 @@ public class SendLogToChainsaw extends XMLLayout { public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(SendLogToChainsaw.class.getName(), args); - + SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level); c.processLogFiles(); } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java b/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java index 4ecd6f9..74b03e4 100644 --- a/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java +++ b/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java @@ -21,7 +21,6 @@ import java.util.Iterator; import java.util.Map.Entry; import java.util.Random; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -43,22 +42,22 @@ import org.apache.log4j.Logger; import com.beust.jcommander.Parameter; public class VerifyIngest { - + private static final Logger log = Logger.getLogger(VerifyIngest.class); - + public static int getRow(Key k) { return Integer.parseInt(k.getRow().toString().split("_")[1]); } - + public static int getCol(Key k) { return Integer.parseInt(k.getColumnQualifier().toString().split("_")[1]); } - + public static class Opts extends TestIngest.Opts { @Parameter(names = "-useGet", description = "fetches values one at a time, instead of scanning") public boolean useGet = false; } - + public static void main(String[] args) throws Exception { Opts opts = new Opts(); ScannerOpts scanOpts = new ScannerOpts(); @@ -71,9 +70,9 @@ public class VerifyIngest { Trace.on(name); Trace.currentTrace().data("cmdLine", Arrays.asList(args).toString()); } - + verifyIngest(opts.getConnector(), opts, scanOpts); - + } finally { Trace.off(); } @@ -82,52 +81,52 @@ public class VerifyIngest { public static void verifyIngest(Connector connector, Opts opts, ScannerOpts scanOpts) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { byte[][] bytevals = TestIngest.generateValues(opts.dataSize); - + Authorizations labelAuths = new Authorizations("L1", "L2", "G1", "GROUP2"); connector.securityOperations().changeUserAuthorizations(opts.principal, labelAuths); - + int expectedRow = opts.startRow; int expectedCol = 0; int recsRead = 0; - + long bytesRead = 0; long t1 = System.currentTimeMillis(); - + byte randomValue[] = new byte[opts.dataSize]; Random random = new Random(); - + Key endKey = new Key(new Text("row_" + String.format("%010d", opts.rows + opts.startRow))); - + int errors = 0; - + while (expectedRow < (opts.rows + opts.startRow)) { - + if (opts.useGet) { Text rowKey = new Text("row_" + String.format("%010d", expectedRow + opts.startRow)); Text colf = new Text(opts.columnFamily); Text colq = new Text("col_" + String.format("%07d", expectedCol)); - + Scanner scanner = connector.createScanner("test_ingest", labelAuths); scanner.setBatchSize(1); Key startKey = new Key(rowKey, colf, colq); Range range = new Range(startKey, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL)); scanner.setRange(range); - + byte[] val = null; // t.get(rowKey, column); - + Iterator<Entry<Key,Value>> iter = scanner.iterator(); - + if (iter.hasNext()) { val = iter.next().getValue().get(); } - + byte ev[]; if (opts.random != null) { ev = TestIngest.genRandomValue(random, randomValue, opts.random.intValue(), expectedRow, expectedCol); } else { ev = bytevals[expectedCol % bytevals.length]; } - + if (val == null) { log.error("Did not find " + rowKey + " " + colf + " " + colq); errors++; @@ -140,94 +139,94 @@ public class VerifyIngest { errors++; } } - + expectedCol++; if (expectedCol >= opts.cols) { expectedCol = 0; expectedRow++; } - + } else { - + Key startKey = new Key(new Text("row_" + String.format("%010d", expectedRow))); - + Scanner scanner = connector.createScanner(opts.getTableName(), labelAuths); scanner.setBatchSize(scanOpts.scanBatchSize); scanner.setRange(new Range(startKey, endKey)); for (int j = 0; j < opts.cols; j++) { scanner.fetchColumn(new Text(opts.columnFamily), new Text("col_" + String.format("%07d", j))); } - + int recsReadBefore = recsRead; - + for (Entry<Key,Value> entry : scanner) { - + recsRead++; - + bytesRead += entry.getKey().getLength(); bytesRead += entry.getValue().getSize(); - + int rowNum = getRow(entry.getKey()); int colNum = getCol(entry.getKey()); - + if (rowNum != expectedRow) { log.error("rowNum != expectedRow " + rowNum + " != " + expectedRow); errors++; expectedRow = rowNum; } - + if (colNum != expectedCol) { log.error("colNum != expectedCol " + colNum + " != " + expectedCol + " rowNum : " + rowNum); errors++; } - + if (expectedRow >= (opts.rows + opts.startRow)) { log.error("expectedRow (" + expectedRow + ") >= (ingestArgs.rows + ingestArgs.startRow) (" + (opts.rows + opts.startRow) + "), get batch returned data passed end key"); errors++; break; } - + byte value[]; if (opts.random != null) { value = TestIngest.genRandomValue(random, randomValue, opts.random.intValue(), expectedRow, colNum); } else { value = bytevals[colNum % bytevals.length]; } - + if (entry.getValue().compareTo(value) != 0) { log.error("unexpected value, rowNum : " + rowNum + " colNum : " + colNum); log.error(" saw = " + new String(entry.getValue().get()) + " expected = " + new String(value)); errors++; } - + if (opts.timestamp >= 0 && entry.getKey().getTimestamp() != opts.timestamp) { log.error("unexpected timestamp " + entry.getKey().getTimestamp() + ", rowNum : " + rowNum + " colNum : " + colNum); errors++; } - + expectedCol++; if (expectedCol >= opts.cols) { expectedCol = 0; expectedRow++; } - + } - + if (recsRead == recsReadBefore) { log.warn("Scan returned nothing, breaking..."); break; } - + } } - + long t2 = System.currentTimeMillis(); - + if (errors > 0) { throw new AccumuloException("saw " + errors + " errors "); } - + if (expectedRow != (opts.rows + opts.startRow)) { throw new AccumuloException("Did not read expected number of rows. Saw " + (expectedRow - opts.startRow) + " expected " + opts.rows); } else { @@ -235,5 +234,5 @@ public class VerifyIngest { (int) ((recsRead) / ((t2 - t1) / 1000.0)), bytesRead, (int) (bytesRead / ((t2 - t1) / 1000.0)), (t2 - t1) / 1000.0); } } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java b/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java index 21d9c1f..ab75632 100644 --- a/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java +++ b/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java @@ -19,7 +19,6 @@ package org.apache.accumulo.test.continuous; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FileReader; import java.io.FilenameFilter; import java.io.InputStreamReader; import java.text.SimpleDateFormat; @@ -54,35 +53,36 @@ import com.beust.jcommander.Parameter; * */ public class UndefinedAnalyzer { - + static class UndefinedNode { - + public UndefinedNode(String undef2, String ref2) { this.undef = undef2; this.ref = ref2; } - + String undef; String ref; } - + static class IngestInfo { - + Map<String,TreeMap<Long,Long>> flushes = new HashMap<String,TreeMap<Long,Long>>(); - + public IngestInfo(String logDir) throws Exception { File dir = new File(logDir); File[] ingestLogs = dir.listFiles(new FilenameFilter() { + @Override public boolean accept(File dir, String name) { return name.endsWith("ingest.out"); } }); - + for (File log : ingestLogs) { parseLog(log); } } - + private void parseLog(File log) throws Exception { BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), Constants.UTF8)); String line; @@ -94,56 +94,56 @@ public class UndefinedAnalyzer { String[] tokens = line.split("\\s"); String time = tokens[1]; String uuid = tokens[2]; - + if (flushes.containsKey(uuid)) { System.err.println("WARN Duplicate uuid " + log); return; } - + tm = new TreeMap<Long,Long>(Collections.reverseOrder()); tm.put(0l, Long.parseLong(time)); flushes.put(uuid, tm); break; - + } if (tm == null) { System.err.println("WARN Bad ingest log " + log); return; } - + while ((line = reader.readLine()) != null) { String[] tokens = line.split("\\s"); - + if (!tokens[0].equals("FLUSH")) continue; - + String time = tokens[1]; String count = tokens[4]; - + tm.put(Long.parseLong(count), Long.parseLong(time)); } } finally { reader.close(); } } - + Iterator<Long> getTimes(String uuid, long count) { TreeMap<Long,Long> tm = flushes.get(uuid); - + if (tm == null) return null; - + return tm.tailMap(count).values().iterator(); } } - + static class TabletAssignment { String tablet; String endRow; String prevEndRow; String server; long time; - + TabletAssignment(String tablet, String er, String per, String server, long time) { this.tablet = tablet; this.endRow = er; @@ -151,30 +151,31 @@ public class UndefinedAnalyzer { this.server = server; this.time = time; } - + public boolean contains(String row) { return prevEndRow.compareTo(row) < 0 && endRow.compareTo(row) >= 0; } } - + static class TabletHistory { - + List<TabletAssignment> assignments = new ArrayList<TabletAssignment>(); - + TabletHistory(String tableId, String acuLogDir) throws Exception { File dir = new File(acuLogDir); File[] masterLogs = dir.listFiles(new FilenameFilter() { + @Override public boolean accept(File dir, String name) { return name.matches("master.*debug.log.*"); } }); - + SimpleDateFormat sdf = new SimpleDateFormat("dd HH:mm:ss,SSS yyyy MM"); String currentYear = (Calendar.getInstance().get(Calendar.YEAR)) + ""; String currentMonth = (Calendar.getInstance().get(Calendar.MONTH) + 1) + ""; - + for (File masterLog : masterLogs) { - + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), Constants.UTF8)); String line; try { @@ -183,11 +184,11 @@ public class UndefinedAnalyzer { String[] tokens = line.split("\\s+"); String tablet = tokens[8]; String server = tokens[10]; - + int pos1 = -1; int pos2 = -1; int pos3 = -1; - + for (int i = 0; i < tablet.length(); i++) { if (tablet.charAt(i) == '<' || tablet.charAt(i) == ';') { if (pos1 == -1) { @@ -199,7 +200,7 @@ public class UndefinedAnalyzer { } } } - + if (pos1 > 0 && pos2 > 0 && pos3 == -1) { String tid = tablet.substring(0, pos1); String endRow = tablet.charAt(pos1) == '<' ? "8000000000000000" : tablet.substring(pos1 + 1, pos2); @@ -208,14 +209,14 @@ public class UndefinedAnalyzer { // System.out.println(" "+server+" "+tid+" "+endRow+" "+prevEndRow); Date date = sdf.parse(tokens[0] + " " + tokens[1] + " " + currentYear + " " + currentMonth); // System.out.println(" "+date); - + assignments.add(new TabletAssignment(tablet, endRow, prevEndRow, server, date.getTime())); - + } } else if (!tablet.startsWith("!0")) { System.err.println("Cannot parse tablet " + tablet); } - + } } } finally { @@ -223,29 +224,32 @@ public class UndefinedAnalyzer { } } } - + TabletAssignment findMostRecentAssignment(String row, long time1, long time2) { - + long latest = Long.MIN_VALUE; TabletAssignment ret = null; - + for (TabletAssignment assignment : assignments) { if (assignment.contains(row) && assignment.time <= time2 && assignment.time > latest) { latest = assignment.time; ret = assignment; } } - + return ret; } } - + static class Opts extends ClientOnDefaultTable { - @Parameter(names="--logdir", description="directory containing the log files", required=true) + @Parameter(names = "--logdir", description = "directory containing the log files", required = true) String logDir; - Opts() { super("ci"); } + + Opts() { + super("ci"); + } } - + /** * Class to analyze undefined references and accumulo logs to isolate the time/tablet where data was lost. */ @@ -253,31 +257,31 @@ public class UndefinedAnalyzer { Opts opts = new Opts(); BatchScannerOpts bsOpts = new BatchScannerOpts(); opts.parseArgs(UndefinedAnalyzer.class.getName(), args, opts); - + List<UndefinedNode> undefs = new ArrayList<UndefinedNode>(); - + BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, Constants.UTF8)); String line; while ((line = reader.readLine()) != null) { String[] tokens = line.split("\\s"); String undef = tokens[0]; String ref = tokens[1]; - + undefs.add(new UndefinedNode(undef, ref)); } - + Connector conn = opts.getConnector(); BatchScanner bscanner = conn.createBatchScanner(opts.getTableName(), opts.auths, bsOpts.scanThreads); bscanner.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS); List<Range> refs = new ArrayList<Range>(); - + for (UndefinedNode undefinedNode : undefs) refs.add(new Range(new Text(undefinedNode.ref))); - + bscanner.setRanges(refs); - + HashMap<String,List<String>> refInfo = new HashMap<String,List<String>>(); - + for (Entry<Key,Value> entry : bscanner) { String ref = entry.getKey().getRow().toString(); List<String> vals = refInfo.get(ref); @@ -285,32 +289,32 @@ public class UndefinedAnalyzer { vals = new ArrayList<String>(); refInfo.put(ref, vals); } - + vals.add(entry.getValue().toString()); } - + bscanner.close(); - + IngestInfo ingestInfo = new IngestInfo(opts.logDir); TabletHistory tabletHistory = new TabletHistory(Tables.getTableId(conn.getInstance(), opts.getTableName()), opts.logDir); - + SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); - + for (UndefinedNode undefinedNode : undefs) { - + List<String> refVals = refInfo.get(undefinedNode.ref); if (refVals != null) { for (String refVal : refVals) { TabletAssignment ta = null; - + String[] tokens = refVal.split(":"); - + String uuid = tokens[0]; String count = tokens[1]; - + String t1 = ""; String t2 = ""; - + Iterator<Long> times = ingestInfo.getTimes(uuid, Long.parseLong(count, 16)); if (times != null) { if (times.hasNext()) { @@ -323,19 +327,19 @@ public class UndefinedAnalyzer { } } } - + if (ta == null) System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + uuid + " " + t1 + " " + t2); else System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + ta.tablet + " " + ta.server + " " + uuid + " " + t1 + " " + t2); - + } } else { System.out.println(undefinedNode.undef + " " + undefinedNode.ref); } - + } - + } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7820120f/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java b/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java index cc183cd..d86742c 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java @@ -24,8 +24,6 @@ import java.util.Map.Entry; import java.util.TreeSet; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector;
