http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/core/src/test/java/org/apache/accumulo/core/util/LocalityGroupUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/LocalityGroupUtilTest.java b/core/src/test/java/org/apache/accumulo/core/util/LocalityGroupUtilTest.java index 333ab83..32d5f62 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/LocalityGroupUtilTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/LocalityGroupUtilTest.java @@ -16,12 +16,15 @@ */ package org.apache.accumulo.core.util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.util.HashSet; import java.util.Map; import java.util.Set; -import static org.junit.Assert.*; - import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; @@ -30,10 +33,10 @@ import org.apache.hadoop.io.Text; import org.junit.Test; public class LocalityGroupUtilTest { - + @Test public void testColumnFamilySet() { - + ConfigurationCopy conf = new ConfigurationCopy(); conf.set("table.group.lg1", "cf1,cf2"); conf.set("table.groups.enabled", "lg1"); @@ -53,7 +56,7 @@ public class LocalityGroupUtilTest { fail(); } catch (LocalityGroupConfigurationError err) {} } - + @Test public void testEncoding() throws Exception { byte test1[] = new byte[256]; @@ -62,18 +65,18 @@ public class LocalityGroupUtilTest { test1[i] = (byte) (0xff & i); test2[i] = (byte) (0xff & (255 - i)); } - + ArrayByteSequence bs1 = new ArrayByteSequence(test1); - + String ecf = LocalityGroupUtil.encodeColumnFamily(bs1); - + // System.out.println(ecf); - + ByteSequence bs2 = LocalityGroupUtil.decodeColumnFamily(ecf); - + assertEquals(bs1, bs2); assertEquals(ecf, LocalityGroupUtil.encodeColumnFamily(bs2)); - + // test encoding multiple column fams containing binary data HashSet<Text> in = new HashSet<Text>(); HashSet<ByteSequence> in2 = new HashSet<ByteSequence>(); @@ -82,8 +85,8 @@ public class LocalityGroupUtilTest { in.add(new Text(test2)); in2.add(new ArrayByteSequence(test2)); Set<ByteSequence> out = LocalityGroupUtil.decodeColumnFamilies(LocalityGroupUtil.encodeColumnFamilies(in)); - + assertEquals(in2, out); } - + }
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/core/src/test/java/org/apache/accumulo/core/util/MergeTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/MergeTest.java b/core/src/test/java/org/apache/accumulo/core/util/MergeTest.java index 544e9e4..95d7b66 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/MergeTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/MergeTest.java @@ -30,11 +30,11 @@ import org.apache.hadoop.io.Text; import org.junit.Test; public class MergeTest { - + static class MergeTester extends Merge { public List<List<Size>> merges = new ArrayList<List<Size>>(); public List<Size> tablets = new ArrayList<Size>(); - + MergeTester(Integer... sizes) { Text start = null; for (Integer size : sizes) { @@ -48,21 +48,21 @@ public class MergeTest { tablets.add(new Size(extent, size)); } } - + @Override protected void message(String format, Object... args) {} - + @Override protected Iterator<Size> getSizeIterator(Connector conn, String tablename, final Text start, final Text end) throws MergeException { final Iterator<Size> impl = tablets.iterator(); return new Iterator<Size>() { Size next = skip(); - + @Override public boolean hasNext() { return next != null; } - + private Size skip() { while (impl.hasNext()) { Size candidate = impl.next(); @@ -78,21 +78,21 @@ public class MergeTest { } return null; } - + @Override public Size next() { Size result = next; next = skip(); return result; } - + @Override public void remove() { impl.remove(); } }; } - + @Override protected void merge(Connector conn, String table, List<Size> sizes, int numToMerge) throws MergeException { List<Size> merge = new ArrayList<Size>(); @@ -102,7 +102,7 @@ public class MergeTest { merges.add(merge); } } - + static private int[] sizes(List<Size> sizes) { int result[] = new int[sizes.size()]; int i = 0; @@ -111,7 +111,7 @@ public class MergeTest { } return result; } - + @Test public void testMergomatic() throws Exception { // Merge everything to the last tablet @@ -120,14 +120,14 @@ public class MergeTest { test.mergomatic(null, "table", null, null, 1000, false); assertEquals(1, test.merges.size()); assertArrayEquals(new int[] {10, 20, 30}, sizes(test.merges.get(i = 0))); - + // Merge ranges around tablets that are big enough test = new MergeTester(1, 2, 100, 1000, 17, 1000, 4, 5, 6, 900); test.mergomatic(null, "table", null, null, 1000, false); assertEquals(2, test.merges.size()); assertArrayEquals(new int[] {1, 2, 100}, sizes(test.merges.get(i = 0))); assertArrayEquals(new int[] {4, 5, 6, 900}, sizes(test.merges.get(++i))); - + // Test the force option test = new MergeTester(1, 2, 100, 1000, 17, 1000, 4, 5, 6, 900); test.mergomatic(null, "table", null, null, 1000, true); @@ -135,25 +135,25 @@ public class MergeTest { assertArrayEquals(new int[] {1, 2, 100}, sizes(test.merges.get(i = 0))); assertArrayEquals(new int[] {17, 1000}, sizes(test.merges.get(++i))); assertArrayEquals(new int[] {4, 5, 6, 900}, sizes(test.merges.get(++i))); - + // Limit the low-end of the merges test = new MergeTester(1, 2, 1000, 17, 1000, 4, 5, 6, 900); test.mergomatic(null, "table", new Text("00004"), null, 1000, false); assertEquals(1, test.merges.size()); assertArrayEquals(new int[] {4, 5, 6, 900}, sizes(test.merges.get(i = 0))); - + // Limit the upper end of the merges test = new MergeTester(1, 2, 1000, 17, 1000, 4, 5, 6, 900); test.mergomatic(null, "table", null, new Text("00004"), 1000, false); assertEquals(1, test.merges.size()); assertArrayEquals(new int[] {1, 2}, sizes(test.merges.get(i = 0))); - + // Limit both ends test = new MergeTester(1, 2, 1000, 17, 1000, 4, 5, 6, 900); test.mergomatic(null, "table", new Text("00002"), new Text("00004"), 1000, true); assertEquals(1, test.merges.size()); assertArrayEquals(new int[] {17, 1000}, sizes(test.merges.get(i = 0))); - + // Clump up tablets into larger values test = new MergeTester(100, 250, 500, 600, 100, 200, 500, 200); test.mergomatic(null, "table", null, null, 1000, false); @@ -162,5 +162,5 @@ public class MergeTest { assertArrayEquals(new int[] {600, 100, 200}, sizes(test.merges.get(++i))); assertArrayEquals(new int[] {500, 200}, sizes(test.merges.get(++i))); } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/core/src/test/java/org/apache/accumulo/core/util/PairTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/PairTest.java b/core/src/test/java/org/apache/accumulo/core/util/PairTest.java index b837f05..60af90e 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/PairTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/PairTest.java @@ -102,15 +102,15 @@ public class PairTest { public void testFromEntry() { Entry<Integer,String> entry = new SimpleImmutableEntry<Integer,String>(10, "IO"); - Pair<Integer, String> pair0 = Pair.fromEntry(entry); + Pair<Integer,String> pair0 = Pair.fromEntry(entry); assertEquals(entry.getKey(), pair0.getFirst()); assertEquals(entry.getValue(), pair0.getSecond()); - Pair<Object,Object> pair = Pair.<Object, Object, Integer, String>fromEntry(entry); + Pair<Object,Object> pair = Pair.<Object,Object,Integer,String> fromEntry(entry); assertEquals(entry.getKey(), pair.getFirst()); assertEquals(entry.getValue(), pair.getSecond()); - Pair<Number,CharSequence> pair2 = Pair.<Number, CharSequence, Integer, String>fromEntry(entry); + Pair<Number,CharSequence> pair2 = Pair.<Number,CharSequence,Integer,String> fromEntry(entry); assertEquals(entry.getKey(), pair2.getFirst()); assertEquals(entry.getValue(), pair2.getSecond()); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/core/src/test/java/org/apache/accumulo/core/util/PartitionerTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/PartitionerTest.java b/core/src/test/java/org/apache/accumulo/core/util/PartitionerTest.java index 276720c..c4538ab 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/PartitionerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/PartitionerTest.java @@ -37,88 +37,88 @@ import org.junit.Test; public class PartitionerTest { @Test public void test1() { - + @SuppressWarnings("unchecked") Map<ByteSequence,MutableLong>[] groups = new Map[2]; - + groups[0] = new HashMap<ByteSequence,MutableLong>(); groups[0].put(new ArrayByteSequence("cf1"), new MutableLong(1)); groups[0].put(new ArrayByteSequence("cf2"), new MutableLong(1)); - + groups[1] = new HashMap<ByteSequence,MutableLong>(); groups[1].put(new ArrayByteSequence("cf3"), new MutableLong(1)); - + Partitioner p1 = new Partitioner(groups); - + Mutation m1 = new Mutation("r1"); m1.put("cf1", "cq1", "v1"); - + Mutation m2 = new Mutation("r2"); m2.put("cf1", "cq1", "v2"); m2.put("cf2", "cq2", "v3"); - + Mutation m3 = new Mutation("r3"); m3.put("cf1", "cq1", "v4"); m3.put("cf3", "cq2", "v5"); - + Mutation m4 = new Mutation("r4"); m4.put("cf1", "cq1", "v6"); m4.put("cf3", "cq2", "v7"); m4.put("cf5", "cq3", "v8"); - + Mutation m5 = new Mutation("r5"); m5.put("cf5", "cq3", "v9"); - + List<Mutation> mutations = Arrays.asList(m1, m2, m3, m4, m5); @SuppressWarnings("unchecked") List<Mutation>[] partitioned = new List[3]; - + for (int i = 0; i < partitioned.length; i++) { partitioned[i] = new ArrayList<Mutation>(); } - + p1.partition(mutations, partitioned); - + m1 = new Mutation("r1"); m1.put("cf1", "cq1", "v1"); - + m2 = new Mutation("r2"); m2.put("cf1", "cq1", "v2"); m2.put("cf2", "cq2", "v3"); - + m3 = new Mutation("r3"); m3.put("cf1", "cq1", "v4"); - + m4 = new Mutation("r4"); m4.put("cf1", "cq1", "v6"); - - Assert.assertEquals(toKeySet(m1,m2,m3,m4), toKeySet(partitioned[0])); - + + Assert.assertEquals(toKeySet(m1, m2, m3, m4), toKeySet(partitioned[0])); + m3 = new Mutation("r3"); m3.put("cf3", "cq2", "v5"); - + m4 = new Mutation("r4"); m4.put("cf3", "cq2", "v7"); - - Assert.assertEquals(toKeySet(m3,m4), toKeySet(partitioned[1])); - + + Assert.assertEquals(toKeySet(m3, m4), toKeySet(partitioned[1])); + m4 = new Mutation("r4"); m4.put("cf5", "cq3", "v8"); - - Assert.assertEquals(toKeySet(m4,m5), toKeySet(partitioned[2])); - + + Assert.assertEquals(toKeySet(m4, m5), toKeySet(partitioned[2])); + } - private Set<Key> toKeySet(List<Mutation> mutations){ + private Set<Key> toKeySet(List<Mutation> mutations) { return toKeySet(mutations.toArray(new Mutation[0])); } - - private Set<Key> toKeySet(Mutation ... expected) { + + private Set<Key> toKeySet(Mutation... expected) { HashSet<Key> ret = new HashSet<Key>(); - for (Mutation mutation : expected) - for(ColumnUpdate cu : mutation.getUpdates()) - ret.add(new Key(mutation.getRow(), cu.getColumnFamily(), cu.getColumnQualifier(), cu.getColumnVisibility(), cu.getTimestamp())); - + for (Mutation mutation : expected) + for (ColumnUpdate cu : mutation.getUpdates()) + ret.add(new Key(mutation.getRow(), cu.getColumnFamily(), cu.getColumnQualifier(), cu.getColumnVisibility(), cu.getTimestamp())); + return ret; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/core/src/test/java/org/apache/accumulo/core/util/TestVersion.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/TestVersion.java b/core/src/test/java/org/apache/accumulo/core/util/TestVersion.java index 6783548..af3f391 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/TestVersion.java +++ b/core/src/test/java/org/apache/accumulo/core/util/TestVersion.java @@ -22,10 +22,10 @@ public class TestVersion extends TestCase { Version make(String version) { return new Version(version); } - + public void testOne() { Version v; - + v = make("abc-1.2.3-ugly"); assertTrue(v != null); assertTrue(v.getPackage().equals("abc")); @@ -33,32 +33,32 @@ public class TestVersion extends TestCase { assertTrue(v.getMinorVersion() == 2); assertTrue(v.getReleaseVersion() == 3); assertTrue(v.getEtcetera().equals("ugly")); - + v = make("3.2.1"); assertTrue(v.getPackage() == null); assertTrue(v.getMajorVersion() == 3); assertTrue(v.getMinorVersion() == 2); assertTrue(v.getReleaseVersion() == 1); assertTrue(v.getEtcetera() == null); - + v = make("55"); assertTrue(v.getPackage() == null); assertTrue(v.getMajorVersion() == 55); assertTrue(v.getMinorVersion() == 0); assertTrue(v.getReleaseVersion() == 0); assertTrue(v.getEtcetera() == null); - + v = make("7.1-beta"); assertTrue(v.getPackage() == null); assertTrue(v.getMajorVersion() == 7); assertTrue(v.getMinorVersion() == 1); assertTrue(v.getReleaseVersion() == 0); assertTrue(v.getEtcetera().equals("beta")); - + try { make("beta"); fail("Should have thrown an error"); } catch (IllegalArgumentException t) {} } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/core/src/test/java/org/apache/accumulo/core/util/TextUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/TextUtilTest.java b/core/src/test/java/org/apache/accumulo/core/util/TextUtilTest.java index b592626..cc45526 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/TextUtilTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/TextUtilTest.java @@ -22,7 +22,7 @@ import org.apache.hadoop.io.Text; /** * Test the TextUtil class. - * + * */ public class TextUtilTest extends TestCase { /** @@ -40,5 +40,5 @@ public class TextUtilTest extends TestCase { assertTrue(TextUtil.getBytes(someText).length == smallerMessage.length()); assertTrue((new Text(TextUtil.getBytes(someText))).equals(smallerMessageText)); } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java index dd4d93f..7b654d0 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java @@ -51,7 +51,7 @@ public class DefaultFormatterTest { @Test public void testAppendBytes() { StringBuilder sb = new StringBuilder(); - byte[] data = new byte[] { 0, '\\', 'x', -0x01 }; + byte[] data = new byte[] {0, '\\', 'x', -0x01}; DefaultFormatter.appendValue(sb, new Value()); assertEquals("", sb.toString()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java index 7f4f3e5..64a3f1b 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java @@ -54,7 +54,6 @@ public class HexFormatterTest { assertEquals(new Text("\0"), formatter.interpretRow(new Text("0"))); } - @Test public void testRoundTripRows() { Text bytes = new Text(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java index 893ed3f..fb460f6 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java @@ -23,7 +23,7 @@ import org.apache.accumulo.core.client.Connector; * Simple example for using tableOperations() (like create, delete, flush, etc). */ public class Flush { - + public static void main(String[] args) { ClientOnRequiredTable opts = new ClientOnRequiredTable(); opts.parseArgs(Flush.class.getName(), args); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java index b34261c..e984f1e 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java @@ -45,33 +45,33 @@ import com.beust.jcommander.Parameter; */ class CountingVerifyingReceiver { private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class); - + long count = 0; int expectedValueSize = 0; HashMap<Text,Boolean> expectedRows; - + CountingVerifyingReceiver(HashMap<Text,Boolean> expectedRows, int expectedValueSize) { this.expectedRows = expectedRows; this.expectedValueSize = expectedValueSize; } - + public void receive(Key key, Value value) { - + String row = key.getRow().toString(); long rowid = Integer.parseInt(row.split("_")[1]); - + byte expectedValue[] = RandomBatchWriter.createValue(rowid, expectedValueSize); - + if (!Arrays.equals(expectedValue, value.get())) { log.error("Got unexpected value for " + key + " expected : " + new String(expectedValue) + " got : " + new String(value.get())); } - + if (!expectedRows.containsKey(key.getRow())) { log.error("Got unexpected key " + key); } else { expectedRows.put(key.getRow(), true); } - + count++; } } @@ -81,10 +81,10 @@ class CountingVerifyingReceiver { */ public class RandomBatchScanner { private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class); - + /** * Generate a number of ranges, each covering a single random row. - * + * * @param num * the number of ranges to generate * @param min @@ -102,20 +102,20 @@ public class RandomBatchScanner { log.info(String.format("Generating %,d random queries...", num)); while (ranges.size() < num) { long rowid = (abs(r.nextLong()) % (max - min)) + min; - + Text row1 = new Text(String.format("row_%010d", rowid)); - + Range range = new Range(new Text(row1)); ranges.add(range); expectedRows.put(row1, false); } - + log.info("finished"); } - + /** * Prints a count of the number of rows mapped to false. - * + * * @return boolean indicating "were all the rows found?" */ private static boolean checkAllRowsFound(HashMap<Text,Boolean> expectedRows) { @@ -124,18 +124,18 @@ public class RandomBatchScanner { for (Entry<Text,Boolean> entry : expectedRows.entrySet()) if (!entry.getValue()) count++; - + if (count > 0) { log.warn("Did not find " + count + " rows"); allFound = false; } return allFound; } - + /** * Generates a number of random queries, verifies that the key/value pairs returned were in the queried ranges and that the values were generated by * {@link RandomBatchWriter#createValue(long, int)}. Prints information about the results. - * + * * @param num * the number of queries to generate * @param min @@ -151,43 +151,43 @@ public class RandomBatchScanner { * @return boolean indicating "did the queries go fine?" */ static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) { - + HashSet<Range> ranges = new HashSet<Range>(num); HashMap<Text,Boolean> expectedRows = new java.util.HashMap<Text,Boolean>(); - + generateRandomQueries(num, min, max, r, ranges, expectedRows); - + tsbr.setRanges(ranges); - + CountingVerifyingReceiver receiver = new CountingVerifyingReceiver(expectedRows, evs); - + long t1 = System.currentTimeMillis(); - + for (Entry<Key,Value> entry : tsbr) { receiver.receive(entry.getKey(), entry.getValue()); } - + long t2 = System.currentTimeMillis(); - + log.info(String.format("%6.2f lookups/sec %6.2f secs%n", num / ((t2 - t1) / 1000.0), ((t2 - t1) / 1000.0))); log.info(String.format("num results : %,d%n", receiver.count)); - + return checkAllRowsFound(expectedRows); } - - public static class Opts extends ClientOnRequiredTable { - @Parameter(names="--min", description="miniumum row that will be generated") + + public static class Opts extends ClientOnRequiredTable { + @Parameter(names = "--min", description = "miniumum row that will be generated") long min = 0; - @Parameter(names="--max", description="maximum ow that will be generated") + @Parameter(names = "--max", description = "maximum ow that will be generated") long max = 0; - @Parameter(names="--num", required=true, description="number of ranges to generate") + @Parameter(names = "--num", required = true, description = "number of ranges to generate") int num = 0; - @Parameter(names="--size", required=true, description="size of the value to write") + @Parameter(names = "--size", required = true, description = "size of the value to write") int size = 0; - @Parameter(names="--seed", description="seed for pseudo-random number generator") + @Parameter(names = "--seed", description = "seed for pseudo-random number generator") Long seed = null; } - + /** * Scans over a specified number of entries to Accumulo using a {@link BatchScanner}. Completes scans twice to compare times for a fresh query with those for * a repeated query which has cached metadata and connections already established. @@ -196,32 +196,32 @@ public class RandomBatchScanner { Opts opts = new Opts(); BatchScannerOpts bsOpts = new BatchScannerOpts(); opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts); - + Connector connector = opts.getConnector(); BatchScanner batchReader = connector.createBatchScanner(opts.getTableName(), opts.auths, bsOpts.scanThreads); batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS); - + Random r; if (opts.seed == null) r = new Random(); else r = new Random(opts.seed); - + // do one cold boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader); - + System.gc(); System.gc(); System.gc(); - + if (opts.seed == null) r = new Random(); else r = new Random(opts.seed); - + // do one hot (connections already established, metadata table cached) status = status && doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader); - + batchReader.close(); if (!status) { System.exit(1); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java index fd56a46..b4d92c8 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java @@ -41,7 +41,7 @@ import com.beust.jcommander.Parameter; /** * Simple example for writing random data to Accumulo. See docs/examples/README.batch for instructions. - * + * * The rows of the entries will be randomly generated numbers between a specified min and max (prefixed by "row_"). The column families will be "foo" and column * qualifiers will be "1". The values will be random byte arrays of a specified size. */ @@ -49,7 +49,7 @@ public class RandomBatchWriter { /** * Creates a random byte array of specified size using the specified seed. - * + * * @param rowid * the seed to use for the random number generator * @param dataSize @@ -72,7 +72,7 @@ public class RandomBatchWriter { /** * Creates a mutation on a specified row with column family "foo", column qualifier "1", specified visibility, and a random value of specified size. - * + * * @param rowid * the row of the mutation * @param dataSize @@ -111,7 +111,7 @@ public class RandomBatchWriter { } public static long abs(long l) { - l = Math.abs(l); // abs(Long.MIN_VALUE) == Long.MIN_VALUE... + l = Math.abs(l); // abs(Long.MIN_VALUE) == Long.MIN_VALUE... if (l < 0) return 0; return l; http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java index ccc924b..b270a6f 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java @@ -64,7 +64,7 @@ public class ReadWriteExample { boolean readEntries = false; @Parameter(names = {"-d", "--delete"}, description = "delete entries after any creates") boolean deleteEntries = false; - @Parameter(names = {"--durability"}, description = "durability used for writes (none, log, flush or sync)", converter=DurabilityConverter.class) + @Parameter(names = {"--durability"}, description = "durability used for writes (none, log, flush or sync)", converter = DurabilityConverter.class) Durability durability = Durability.DEFAULT; public Opts() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java index 5276109..ba7f9ad 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java @@ -41,135 +41,135 @@ import org.apache.log4j.Logger; * A demonstration of reading entire rows and deleting entire rows. */ public class RowOperations { - + private static final Logger log = Logger.getLogger(RowOperations.class); - + private static Connector connector; private static String table = "example"; private static BatchWriter bw; - + public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, MutationsRejectedException { - + ClientOpts opts = new ClientOpts(); ScannerOpts scanOpts = new ScannerOpts(); BatchWriterOpts bwOpts = new BatchWriterOpts(); opts.parseArgs(RowOperations.class.getName(), args, scanOpts, bwOpts); - + // First the setup work connector = opts.getConnector(); - + // lets create an example table connector.tableOperations().create(table); - + // lets create 3 rows of information Text row1 = new Text("row1"); Text row2 = new Text("row2"); Text row3 = new Text("row3"); - + // Which means 3 different mutations Mutation mut1 = new Mutation(row1); Mutation mut2 = new Mutation(row2); Mutation mut3 = new Mutation(row3); - + // And we'll put 4 columns in each row Text col1 = new Text("1"); Text col2 = new Text("2"); Text col3 = new Text("3"); Text col4 = new Text("4"); - + // Now we'll add them to the mutations mut1.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut1.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut1.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut1.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); - + mut2.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut2.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut2.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut2.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); - + mut3.put(new Text("column"), col1, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut3.put(new Text("column"), col2, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut3.put(new Text("column"), col3, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); mut3.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes())); - + // Now we'll make a Batch Writer bw = connector.createBatchWriter(table, bwOpts.getBatchWriterConfig()); - + // And add the mutations bw.addMutation(mut1); bw.addMutation(mut2); bw.addMutation(mut3); - + // Force a send bw.flush(); - + // Now lets look at the rows Scanner rowThree = getRow(scanOpts, new Text("row3")); Scanner rowTwo = getRow(scanOpts, new Text("row2")); Scanner rowOne = getRow(scanOpts, new Text("row1")); - + // And print them log.info("This is everything"); printRow(rowOne); printRow(rowTwo); printRow(rowThree); System.out.flush(); - + // Now lets delete rowTwo with the iterator rowTwo = getRow(scanOpts, new Text("row2")); deleteRow(rowTwo); - + // Now lets look at the rows again rowThree = getRow(scanOpts, new Text("row3")); rowTwo = getRow(scanOpts, new Text("row2")); rowOne = getRow(scanOpts, new Text("row1")); - + // And print them log.info("This is row1 and row3"); printRow(rowOne); printRow(rowTwo); printRow(rowThree); System.out.flush(); - + // Should only see the two rows // Now lets delete rowOne without passing in the iterator - + deleteRow(scanOpts, row1); - + // Now lets look at the rows one last time rowThree = getRow(scanOpts, new Text("row3")); rowTwo = getRow(scanOpts, new Text("row2")); rowOne = getRow(scanOpts, new Text("row1")); - + // And print them log.info("This is just row3"); printRow(rowOne); printRow(rowTwo); printRow(rowThree); System.out.flush(); - + // Should only see rowThree - + // Always close your batchwriter - + bw.close(); - + // and lets clean up our mess connector.tableOperations().delete(table); - + // fin~ - + } - + /** * Deletes a row given a text object */ private static void deleteRow(ScannerOpts scanOpts, Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { deleteRow(getRow(scanOpts, row)); } - + /** * Deletes a row, given a Scanner of JUST that row */ @@ -186,7 +186,7 @@ public class RowOperations { bw.addMutation(deleter); bw.flush(); } - + /** * Just a generic print function given an iterator. Not necessarily just for printing a single row */ @@ -195,7 +195,7 @@ public class RowOperations { for (Entry<Key,Value> entry : scanner) log.info("Key: " + entry.getKey().toString() + " Value: " + entry.getValue().toString()); } - + /** * Gets a scanner over one row */ @@ -208,5 +208,5 @@ public class RowOperations { scanner.setRange(new Range(row)); return scanner; } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java index 3ae21e9..f2bd4d7 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java @@ -33,18 +33,18 @@ import com.beust.jcommander.Parameter; * Simple example for writing random data in sequential order to Accumulo. See docs/examples/README.batch for instructions. */ public class SequentialBatchWriter { - + static class Opts extends ClientOnRequiredTable { - @Parameter(names="--start") + @Parameter(names = "--start") long start = 0; - @Parameter(names="--num", required=true) + @Parameter(names = "--num", required = true) long num = 0; - @Parameter(names="--size", required=true, description="size of the value to write") + @Parameter(names = "--size", required = true, description = "size of the value to write") int valueSize = 0; - @Parameter(names="--vis", converter=VisibilityConverter.class) + @Parameter(names = "--vis", converter = VisibilityConverter.class) ColumnVisibility vis = new ColumnVisibility(); } - + /** * Writes a specified number of entries to Accumulo using a {@link BatchWriter}. The rows of the entries will be sequential starting at a specified number. * The column families will be "foo" and column qualifiers will be "1". The values will be random byte arrays of a specified size. @@ -55,14 +55,14 @@ public class SequentialBatchWriter { opts.parseArgs(SequentialBatchWriter.class.getName(), args, bwOpts); Connector connector = opts.getConnector(); BatchWriter bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig()); - + long end = opts.start + opts.num; - + for (long i = opts.start; i < end; i++) { Mutation m = RandomBatchWriter.createMutation(i, opts.valueSize, opts.vis); bw.addMutation(m); } - + bw.close(); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java index 8fbf5e9..46e6a67 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java @@ -34,39 +34,39 @@ import com.beust.jcommander.Parameter; * */ public class TraceDumpExample { - - static class Opts extends ClientOnDefaultTable { - public Opts() { - super("trace"); - } - @Parameter(names = {"--traceid"}, description = "The hex string id of a given trace, for example 16cfbbd7beec4ae3") - public String traceId = ""; - } - - public void dump(Opts opts) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { - - if (opts.traceId.isEmpty()) { - throw new IllegalArgumentException("--traceid option is required"); - } - - Scanner scanner = opts.getConnector().createScanner(opts.getTableName(), opts.auths); - scanner.setRange(new Range(new Text(opts.traceId))); - TraceDump.printTrace(scanner, new Printer() { - @Override + static class Opts extends ClientOnDefaultTable { + public Opts() { + super("trace"); + } + + @Parameter(names = {"--traceid"}, description = "The hex string id of a given trace, for example 16cfbbd7beec4ae3") + public String traceId = ""; + } + + public void dump(Opts opts) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + + if (opts.traceId.isEmpty()) { + throw new IllegalArgumentException("--traceid option is required"); + } + + Scanner scanner = opts.getConnector().createScanner(opts.getTableName(), opts.auths); + scanner.setRange(new Range(new Text(opts.traceId))); + TraceDump.printTrace(scanner, new Printer() { + @Override public void print(String line) { - System.out.println(line); - } - }); - } - - public static void main(String[] args) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { - TraceDumpExample traceDumpExample = new TraceDumpExample(); - Opts opts = new Opts(); - ScannerOpts scannerOpts = new ScannerOpts(); - opts.parseArgs(TraceDumpExample.class.getName(), args, scannerOpts); + System.out.println(line); + } + }); + } + + public static void main(String[] args) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + TraceDumpExample traceDumpExample = new TraceDumpExample(); + Opts opts = new Opts(); + ScannerOpts scannerOpts = new ScannerOpts(); + opts.parseArgs(TraceDumpExample.class.getName(), args, scannerOpts); - traceDumpExample.dump(opts); - } + traceDumpExample.dump(opts); + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TracingExample.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TracingExample.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TracingExample.java index 881819b..2c4a8a9 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TracingExample.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TracingExample.java @@ -18,6 +18,7 @@ package org.apache.accumulo.examples.simple.client; import static java.nio.charset.StandardCharsets.UTF_8; + import java.util.Map.Entry; import org.apache.accumulo.core.cli.ClientOnDefaultTable; @@ -43,7 +44,7 @@ import com.beust.jcommander.Parameter; /** * A simple example showing how to use the distributed tracing API in client code - * + * */ public class TracingExample { private static final Logger log = Logger.getLogger(TracingExample.class); @@ -124,8 +125,7 @@ public class TracingExample { ++numberOfEntriesRead; } // You can add additional metadata (key, values) to Spans which will be able to be viewed in the Monitor - readScope.getSpan().addKVAnnotation("Number of Entries Read".getBytes(UTF_8), - String.valueOf(numberOfEntriesRead).getBytes(UTF_8)); + readScope.getSpan().addKVAnnotation("Number of Entries Read".getBytes(UTF_8), String.valueOf(numberOfEntriesRead).getBytes(UTF_8)); readScope.close(); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java index 41a531b..7dad89c 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java @@ -33,22 +33,22 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; * and count. See {@link Combiner} for more information on which values are combined together. See docs/examples/README.combiner for instructions. */ public class StatsCombiner extends Combiner { - + public static final String RADIX_OPTION = "radix"; - + private int radix = 10; - + @Override public Value reduce(Key key, Iterator<Value> iter) { - + long min = Long.MAX_VALUE; long max = Long.MIN_VALUE; long sum = 0; long count = 0; - + while (iter.hasNext()) { String stats[] = iter.next().toString().split(","); - + if (stats.length == 1) { long val = Long.parseLong(stats[0], radix); min = Math.min(val, min); @@ -62,21 +62,21 @@ public class StatsCombiner extends Combiner { count += Long.parseLong(stats[3], radix); } } - + String ret = Long.toString(min, radix) + "," + Long.toString(max, radix) + "," + Long.toString(sum, radix) + "," + Long.toString(count, radix); return new Value(ret.getBytes()); } - + @Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { super.init(source, options, env); - + if (options.containsKey(RADIX_OPTION)) radix = Integer.parseInt(options.get(RADIX_OPTION)); else radix = 10; } - + @Override public IteratorOptions describeOptions() { IteratorOptions io = super.describeOptions(); @@ -85,21 +85,21 @@ public class StatsCombiner extends Combiner { io.addNamedOption(RADIX_OPTION, "radix/base of the numbers"); return io; } - + @Override public boolean validateOptions(Map<String,String> options) { if (!super.validateOptions(options)) return false; - + if (options.containsKey(RADIX_OPTION) && !options.get(RADIX_OPTION).matches("\\d+")) throw new IllegalArgumentException("invalid option " + RADIX_OPTION + ":" + options.get(RADIX_OPTION)); - + return true; } - + /** * A convenience method for setting the expected base/radix of the numbers - * + * * @param iterConfig * Iterator settings to configure * @param base http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraint.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraint.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraint.java index b23a258..8099b7e 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraint.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraint.java @@ -26,27 +26,27 @@ import org.apache.accumulo.core.data.Mutation; /** * This class is an accumulo constraint that ensures all fields of a key are alpha numeric. - * + * * See docs/examples/README.constraint for instructions. - * + * */ public class AlphaNumKeyConstraint implements Constraint { - + private static final short NON_ALPHA_NUM_ROW = 1; private static final short NON_ALPHA_NUM_COLF = 2; private static final short NON_ALPHA_NUM_COLQ = 3; - + private boolean isAlphaNum(byte bytes[]) { for (byte b : bytes) { boolean ok = ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')); if (!ok) return false; } - + return true; } - + private List<Short> addViolation(List<Short> violations, short violation) { if (violations == null) { violations = new ArrayList<Short>(); @@ -56,29 +56,29 @@ public class AlphaNumKeyConstraint implements Constraint { } return violations; } - + @Override public List<Short> check(Environment env, Mutation mutation) { List<Short> violations = null; - + if (!isAlphaNum(mutation.getRow())) violations = addViolation(violations, NON_ALPHA_NUM_ROW); - + Collection<ColumnUpdate> updates = mutation.getUpdates(); for (ColumnUpdate columnUpdate : updates) { if (!isAlphaNum(columnUpdate.getColumnFamily())) violations = addViolation(violations, NON_ALPHA_NUM_COLF); - + if (!isAlphaNum(columnUpdate.getColumnQualifier())) violations = addViolation(violations, NON_ALPHA_NUM_COLQ); } - + return violations; } - + @Override public String getViolationDescription(short violationCode) { - + switch (violationCode) { case NON_ALPHA_NUM_ROW: return "Row was not alpha numeric"; @@ -87,8 +87,8 @@ public class AlphaNumKeyConstraint implements Constraint { case NON_ALPHA_NUM_COLQ: return "Column qualifier was not alpha numeric"; } - + return null; } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/MaxMutationSize.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/MaxMutationSize.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/MaxMutationSize.java index 54a863e..1192549 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/MaxMutationSize.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/MaxMutationSize.java @@ -24,18 +24,18 @@ import org.apache.accumulo.core.data.Mutation; /** * Ensure that mutations are a reasonable size: we must be able to fit several in memory at a time. - * + * */ public class MaxMutationSize implements Constraint { static final long MAX_SIZE = Runtime.getRuntime().maxMemory() >> 8; static final List<Short> empty = Collections.emptyList(); static final List<Short> violations = Collections.singletonList(new Short((short) 0)); - + @Override public String getViolationDescription(short violationCode) { return String.format("mutation exceeded maximum size of %d", MAX_SIZE); } - + @Override public List<Short> check(Environment env, Mutation mutation) { if (mutation.estimatedMemoryUsed() < MAX_SIZE) http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraint.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraint.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraint.java index b7e1527..f1e6d5a 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraint.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraint.java @@ -28,19 +28,19 @@ import org.apache.accumulo.core.data.Mutation; * This class is an accumulo constraint that ensures values are numeric strings. See docs/examples/README.constraint for instructions. */ public class NumericValueConstraint implements Constraint { - + private static final short NON_NUMERIC_VALUE = 1; - + private boolean isNumeric(byte bytes[]) { for (byte b : bytes) { boolean ok = (b >= '0' && b <= '9'); if (!ok) return false; } - + return true; } - + private List<Short> addViolation(List<Short> violations, short violation) { if (violations == null) { violations = new ArrayList<Short>(); @@ -50,30 +50,30 @@ public class NumericValueConstraint implements Constraint { } return violations; } - + @Override public List<Short> check(Environment env, Mutation mutation) { List<Short> violations = null; - + Collection<ColumnUpdate> updates = mutation.getUpdates(); - + for (ColumnUpdate columnUpdate : updates) { if (!isNumeric(columnUpdate.getValue())) violations = addViolation(violations, NON_NUMERIC_VALUE); } - + return violations; } - + @Override public String getViolationDescription(short violationCode) { - + switch (violationCode) { case NON_NUMERIC_VALUE: return "Value is not numeric"; } - + return null; } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java index cb6d350..dabb4c1 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java @@ -38,21 +38,20 @@ import com.beust.jcommander.Parameter; * Computes recursive counts over file system information and stores them back into the same Accumulo table. See docs/examples/README.dirlist for instructions. */ public class FileCount { - + private int entriesScanned; private int inserts; - + private Opts opts; private ScannerOpts scanOpts; private BatchWriterOpts bwOpts; - - + private static class CountValue { int dirCount = 0; int fileCount = 0; int recursiveDirCount = 0; int recusiveFileCount = 0; - + void set(Value val) { String sa[] = val.toString().split(","); dirCount = Integer.parseInt(sa[0]); @@ -60,46 +59,46 @@ public class FileCount { recursiveDirCount = Integer.parseInt(sa[2]); recusiveFileCount = Integer.parseInt(sa[3]); } - + Value toValue() { return new Value((dirCount + "," + fileCount + "," + recursiveDirCount + "," + recusiveFileCount).getBytes()); } - + void incrementFiles() { fileCount++; recusiveFileCount++; } - + void incrementDirs() { dirCount++; recursiveDirCount++; } - + public void clear() { dirCount = 0; fileCount = 0; recursiveDirCount = 0; recusiveFileCount = 0; } - + public void incrementRecursive(CountValue other) { recursiveDirCount += other.recursiveDirCount; recusiveFileCount += other.recusiveFileCount; } } - + private int findMaxDepth(Scanner scanner, int min, int max) { int mid = min + (max - min) / 2; return findMaxDepth(scanner, min, mid, max); } - + private int findMaxDepth(Scanner scanner, int min, int mid, int max) { // check to see if the mid point exist if (max < min) return -1; - + scanner.setRange(new Range(String.format("%03d", mid), true, String.format("%03d", mid + 1), false)); - + if (scanner.iterator().hasNext()) { // this depth exist, check to see if a larger depth exist int ret = findMaxDepth(scanner, mid + 1, max); @@ -111,9 +110,9 @@ public class FileCount { // this depth does not exist, look lower return findMaxDepth(scanner, min, mid - 1); } - + } - + private int findMaxDepth(Scanner scanner) { // do binary search to find max depth int origBatchSize = scanner.getBatchSize(); @@ -122,81 +121,81 @@ public class FileCount { scanner.setBatchSize(origBatchSize); return depth; } - + // find the count column and consume a row private Entry<Key,Value> findCount(Entry<Key,Value> entry, Iterator<Entry<Key,Value>> iterator, CountValue cv) { - + Key key = entry.getKey(); Text currentRow = key.getRow(); - + if (key.compareColumnQualifier(QueryUtil.COUNTS_COLQ) == 0) cv.set(entry.getValue()); - + while (iterator.hasNext()) { entry = iterator.next(); entriesScanned++; key = entry.getKey(); - + if (key.compareRow(currentRow) != 0) return entry; - + if (key.compareColumnFamily(QueryUtil.DIR_COLF) == 0 && key.compareColumnQualifier(QueryUtil.COUNTS_COLQ) == 0) { cv.set(entry.getValue()); } - + } - + return null; } - + private Entry<Key,Value> consumeRow(Entry<Key,Value> entry, Iterator<Entry<Key,Value>> iterator) { Key key = entry.getKey(); Text currentRow = key.getRow(); - + while (iterator.hasNext()) { entry = iterator.next(); entriesScanned++; key = entry.getKey(); - + if (key.compareRow(currentRow) != 0) return entry; } - + return null; } - + private String extractDir(Key key) { String row = key.getRowData().toString(); return row.substring(3, row.lastIndexOf('/')); } - + private Mutation createMutation(int depth, String dir, CountValue countVal) { Mutation m = new Mutation(String.format("%03d%s", depth, dir)); m.put(QueryUtil.DIR_COLF, QueryUtil.COUNTS_COLQ, opts.visibility, countVal.toValue()); return m; } - + private void calculateCounts(Scanner scanner, int depth, BatchWriter batchWriter) throws Exception { - + scanner.setRange(new Range(String.format("%03d", depth), true, String.format("%03d", depth + 1), false)); - + CountValue countVal = new CountValue(); - + Iterator<Entry<Key,Value>> iterator = scanner.iterator(); - + String currentDir = null; - + Entry<Key,Value> entry = null; if (iterator.hasNext()) { entry = iterator.next(); entriesScanned++; } - + while (entry != null) { Key key = entry.getKey(); - + String dir = extractDir(key); - + if (currentDir == null) { currentDir = dir; } else if (!currentDir.equals(dir)) { @@ -205,12 +204,12 @@ public class FileCount { currentDir = dir; countVal.clear(); } - + // process a whole row if (key.compareColumnFamily(QueryUtil.DIR_COLF) == 0) { CountValue tmpCount = new CountValue(); entry = findCount(entry, iterator, tmpCount); - + if (tmpCount.dirCount == 0 && tmpCount.fileCount == 0) { // in this case the higher depth will not insert anything if the // dir has no children, so insert something here @@ -219,7 +218,7 @@ public class FileCount { batchWriter.addMutation(m); inserts++; } - + countVal.incrementRecursive(tmpCount); countVal.incrementDirs(); } else { @@ -227,57 +226,57 @@ public class FileCount { countVal.incrementFiles(); } } - + if (currentDir != null) { batchWriter.addMutation(createMutation(depth - 1, currentDir, countVal)); inserts++; } } - + FileCount(Opts opts, ScannerOpts scanOpts, BatchWriterOpts bwOpts) throws Exception { this.opts = opts; this.scanOpts = scanOpts; this.bwOpts = bwOpts; } - + public void run() throws Exception { - + entriesScanned = 0; inserts = 0; - + Connector conn = opts.getConnector(); Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths); scanner.setBatchSize(scanOpts.scanBatchSize); BatchWriter bw = conn.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig()); - + long t1 = System.currentTimeMillis(); - + int depth = findMaxDepth(scanner); - + long t2 = System.currentTimeMillis(); - + for (int d = depth; d > 0; d--) { calculateCounts(scanner, d, bw); // must flush so next depth can read what prev depth wrote bw.flush(); } - + bw.close(); - + long t3 = System.currentTimeMillis(); - + System.out.printf("Max depth : %d%n", depth); System.out.printf("Time to find max depth : %,d ms%n", (t2 - t1)); System.out.printf("Time to compute counts : %,d ms%n", (t3 - t2)); System.out.printf("Entries scanned : %,d %n", entriesScanned); System.out.printf("Counts inserted : %,d %n", inserts); } - + public static class Opts extends ClientOnRequiredTable { - @Parameter(names="--vis", description="use a given visibility for the new counts", converter=VisibilityConverter.class) + @Parameter(names = "--vis", description = "use a given visibility for the new counts", converter = VisibilityConverter.class) ColumnVisibility visibility = new ColumnVisibility(); } - + public static void main(String[] args) throws Exception { Opts opts = new Opts(); ScannerOpts scanOpts = new ScannerOpts(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java index 0ec10fb..17c9ee8 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java @@ -49,7 +49,7 @@ public class Ingest { public static final String LASTMOD_CQ = "lastmod"; public static final String HASH_CQ = "md5"; public static final Encoder<Long> encoder = LongCombiner.FIXED_LEN_ENCODER; - + public static Mutation buildMutation(ColumnVisibility cv, String path, boolean isDir, boolean isHidden, boolean canExec, long length, long lastmod, String hash) { if (path.equals("/")) @@ -68,7 +68,7 @@ public class Ingest { m.put(colf, new Text(HASH_CQ), cv, new Value(hash.getBytes())); return m; } - + private static void ingest(File src, ColumnVisibility cv, BatchWriter dirBW, BatchWriter indexBW, FileDataIngest fdi, BatchWriter data) throws Exception { // build main table entry String path = null; @@ -78,7 +78,7 @@ public class Ingest { path = src.getAbsolutePath(); } System.out.println(path); - + String hash = null; if (!src.isDirectory()) { try { @@ -88,9 +88,9 @@ public class Ingest { return; } } - + dirBW.addMutation(buildMutation(cv, path, src.isDirectory(), src.isHidden(), src.canExecute(), src.length(), src.lastModified(), hash)); - + // build index table entries Text row = QueryUtil.getForwardIndex(path); if (row != null) { @@ -98,14 +98,14 @@ public class Ingest { Mutation m = new Mutation(row); m.put(QueryUtil.INDEX_COLF, p, cv, nullValue); indexBW.addMutation(m); - + row = QueryUtil.getReverseIndex(path); m = new Mutation(row); m.put(QueryUtil.INDEX_COLF, p, cv, nullValue); indexBW.addMutation(m); } } - + private static void recurse(File src, ColumnVisibility cv, BatchWriter dirBW, BatchWriter indexBW, FileDataIngest fdi, BatchWriter data) throws Exception { // ingest this File ingest(src, cv, dirBW, indexBW, fdi, data); @@ -119,28 +119,27 @@ public class Ingest { } } } - + static class Opts extends ClientOpts { - @Parameter(names="--dirTable", description="a table to hold the directory information") + @Parameter(names = "--dirTable", description = "a table to hold the directory information") String nameTable = "dirTable"; - @Parameter(names="--indexTable", description="an index over the ingested data") + @Parameter(names = "--indexTable", description = "an index over the ingested data") String indexTable = "indexTable"; - @Parameter(names="--dataTable", description="the file data, chunked into parts") + @Parameter(names = "--dataTable", description = "the file data, chunked into parts") String dataTable = "dataTable"; - @Parameter(names="--vis", description="the visibility to mark the data", converter=VisibilityConverter.class) + @Parameter(names = "--vis", description = "the visibility to mark the data", converter = VisibilityConverter.class) ColumnVisibility visibility = new ColumnVisibility(); - @Parameter(names="--chunkSize", description="the size of chunks when breaking down files") + @Parameter(names = "--chunkSize", description = "the size of chunks when breaking down files") int chunkSize = 100000; - @Parameter(description="<dir> { <dir> ... }") + @Parameter(description = "<dir> { <dir> ... }") List<String> directories = new ArrayList<String>(); } - - + public static void main(String[] args) throws Exception { Opts opts = new Opts(); BatchWriterOpts bwOpts = new BatchWriterOpts(); opts.parseArgs(Ingest.class.getName(), args, bwOpts); - + Connector conn = opts.getConnector(); if (!conn.tableOperations().exists(opts.nameTable)) conn.tableOperations().create(opts.nameTable); @@ -150,14 +149,14 @@ public class Ingest { conn.tableOperations().create(opts.dataTable); conn.tableOperations().attachIterator(opts.dataTable, new IteratorSetting(1, ChunkCombiner.class)); } - + BatchWriter dirBW = conn.createBatchWriter(opts.nameTable, bwOpts.getBatchWriterConfig()); BatchWriter indexBW = conn.createBatchWriter(opts.indexTable, bwOpts.getBatchWriterConfig()); BatchWriter dataBW = conn.createBatchWriter(opts.dataTable, bwOpts.getBatchWriterConfig()); FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility); for (String dir : opts.directories) { recurse(new File(dir), opts.visibility, dirBW, indexBW, fdi, dataBW); - + // fill in parent directory info int slashIndex = -1; while ((slashIndex = dir.lastIndexOf("/")) > 0) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java index 09fb40c..a79b9d2 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java @@ -49,17 +49,16 @@ public class QueryUtil { public static final Text REVERSE_PREFIX = new Text("r"); public static final Text INDEX_COLF = new Text("i"); public static final Text COUNTS_COLQ = new Text("counts"); - - public QueryUtil(Opts opts) throws AccumuloException, - AccumuloSecurityException { + + public QueryUtil(Opts opts) throws AccumuloException, AccumuloSecurityException { conn = opts.getConnector(); this.tableName = opts.getTableName(); this.auths = opts.auths; } - + /** * Calculates the depth of a path, i.e. the number of forward slashes in the path name. - * + * * @param path * the full path of a file or directory * @return the depth of the path @@ -71,10 +70,10 @@ public class QueryUtil { numSlashes++; return numSlashes; } - + /** * Given a path, construct an accumulo row prepended with the path's depth for the directory table. - * + * * @param path * the full path of a file or directory * @return the accumulo row associated with this path @@ -84,10 +83,10 @@ public class QueryUtil { row.append(path.getBytes(), 0, path.length()); return row; } - + /** * Given a path, construct an accumulo row prepended with the {@link #FORWARD_PREFIX} for the index table. - * + * * @param path * the full path of a file or directory * @return the accumulo row associated with this path @@ -100,10 +99,10 @@ public class QueryUtil { row.append(part.getBytes(), 0, part.length()); return row; } - + /** * Given a path, construct an accumulo row prepended with the {@link #REVERSE_PREFIX} with the path reversed for the index table. - * + * * @param path * the full path of a file or directory * @return the accumulo row associated with this path @@ -120,10 +119,10 @@ public class QueryUtil { row.append(rev, 0, rev.length); return row; } - + /** * Returns either the {@link #DIR_COLF} or a decoded string version of the colf. - * + * * @param colf * the column family */ @@ -132,10 +131,10 @@ public class QueryUtil { return colf.toString() + ":"; return Long.toString(Ingest.encoder.decode(colf.getBytes())) + ":"; } - + /** * Scans over the directory table and pulls out stat information about a path. - * + * * @param path * the full path of a file or directory */ @@ -152,10 +151,10 @@ public class QueryUtil { } return data; } - + /** * Uses the directory table to list the contents of a directory. - * + * * @param path * the full path of a directory */ @@ -177,10 +176,10 @@ public class QueryUtil { } return fim; } - + /** * Scans over the index table for files or directories with a given name. - * + * * @param term * the name a file or directory to search for */ @@ -190,17 +189,17 @@ public class QueryUtil { scanner.setRange(new Range(getForwardIndex(term))); return scanner; } - + /** * Scans over the index table for files or directories with a given name, prefix, or suffix (indicated by a wildcard '*' at the beginning or end of the term. - * + * * @param exp * the name a file or directory to search for with an optional wildcard '*' at the beginning or end */ public Iterable<Entry<Key,Value>> singleRestrictedWildCardSearch(String exp) throws Exception { if (exp.indexOf("/") >= 0) throw new Exception("this method only works with unqualified names"); - + Scanner scanner = conn.createScanner(tableName, auths); if (exp.startsWith("*")) { System.out.println("executing beginning wildcard search for " + exp); @@ -217,10 +216,10 @@ public class QueryUtil { } return scanner; } - + /** * Scans over the index table for files or directories with a given name that can contain a single wildcard '*' anywhere in the term. - * + * * @param exp * the name a file or directory to search for with one optional wildcard '*' */ @@ -228,17 +227,17 @@ public class QueryUtil { int starIndex = exp.indexOf("*"); if (exp.indexOf("*", starIndex + 1) >= 0) throw new Exception("only one wild card for search"); - + if (starIndex < 0) { return exactTermSearch(exp); } else if (starIndex == 0 || starIndex == exp.length() - 1) { return singleRestrictedWildCardSearch(exp); } - + String firstPart = exp.substring(0, starIndex); String lastPart = exp.substring(starIndex + 1); String regexString = ".*/" + exp.replace("*", "[^/]*"); - + Scanner scanner = conn.createScanner(tableName, auths); if (firstPart.length() >= lastPart.length()) { System.out.println("executing middle wildcard search for " + regexString + " from entries starting with " + firstPart); @@ -252,14 +251,14 @@ public class QueryUtil { scanner.addScanIterator(regex); return scanner; } - + public static class Opts extends ClientOnRequiredTable { - @Parameter(names="--path", description="the directory to list") + @Parameter(names = "--path", description = "the directory to list") String path = "/"; - @Parameter(names="--search", description="find a file or directory with the given name") + @Parameter(names = "--search", description = "find a file or directory with the given name") boolean search = false; } - + /** * Lists the contents of a directory using the directory table, or searches for file or directory names (if the -search flag is included). */ http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java index 172ba58..70d989f 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java @@ -47,7 +47,7 @@ import com.beust.jcommander.Parameter; @SuppressWarnings("serial") public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansionListener { private static final Logger log = Logger.getLogger(Viewer.class); - + JTree tree; DefaultTreeModel treeModel; QueryUtil q; @@ -57,35 +57,35 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi JTextArea text; JTextArea data; JScrollPane dataPane; - + public static class NodeInfo { private String name; private Map<String,String> data; - + public NodeInfo(String name, Map<String,String> data) { this.name = name; this.data = data; } - + public String getName() { return name; } - + public String getFullName() { String fn = data.get("fullname"); if (fn == null) return name; return fn; } - + public Map<String,String> getData() { return data; } - + public String toString() { return getName(); } - + public String getHash() { for (String k : data.keySet()) { String[] parts = k.split(":"); @@ -96,9 +96,8 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi return null; } } - - public Viewer(Opts opts) - throws Exception { + + public Viewer(Opts opts) throws Exception { super("File Viewer"); setSize(1000, 800); setDefaultCloseOperation(EXIT_ON_CLOSE); @@ -106,7 +105,7 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi fdq = new FileDataQuery(opts.instance, opts.zookeepers, opts.principal, opts.getToken(), opts.dataTable, opts.auths); this.topPath = opts.path; } - + public void populate(DefaultMutableTreeNode node) throws TableNotFoundException { String path = ((NodeInfo) node.getUserObject()).getFullName(); log.debug("listing " + path); @@ -115,7 +114,7 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi node.add(new DefaultMutableTreeNode(new NodeInfo(e.getKey(), e.getValue()))); } } - + public void populateChildren(DefaultMutableTreeNode node) throws TableNotFoundException { @SuppressWarnings("unchecked") Enumeration<DefaultMutableTreeNode> children = node.children(); @@ -123,12 +122,12 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi populate(children.nextElement()); } } - + public void init() throws TableNotFoundException { DefaultMutableTreeNode root = new DefaultMutableTreeNode(new NodeInfo(topPath, q.getData(topPath))); populate(root); populateChildren(root); - + treeModel = new DefaultTreeModel(root); tree = new JTree(treeModel); tree.addTreeExpansionListener(this); @@ -144,11 +143,11 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi infoSplitPane.setDividerLocation(150); getContentPane().add(mainSplitPane, BorderLayout.CENTER); } - + public static String getText(DefaultMutableTreeNode node) { return getText(((NodeInfo) node.getUserObject()).getData()); } - + public static String getText(Map<String,String> data) { StringBuilder sb = new StringBuilder(); for (String name : data.keySet()) { @@ -159,7 +158,7 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi } return sb.toString(); } - + @Override public void treeExpanded(TreeExpansionEvent event) { try { @@ -168,7 +167,7 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi log.error("Could not find table.", e); } } - + @Override public void treeCollapsed(TreeExpansionEvent event) { DefaultMutableTreeNode node = (DefaultMutableTreeNode) event.getPath().getLastPathComponent(); @@ -180,7 +179,7 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi child.removeAllChildren(); } } - + @Override public void valueChanged(TreeSelectionEvent e) { TreePath selected = e.getNewLeadSelectionPath(); @@ -199,16 +198,16 @@ public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansi log.error("Could not get data from FileDataQuery.", e1); } } - + static class Opts extends QueryUtil.Opts { - @Parameter(names="--dataTable") + @Parameter(names = "--dataTable") String dataTable = "dataTable"; } - + public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(Viewer.class.getName(), args); - + Viewer v = new Viewer(opts); v.init(); v.setVisible(true); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java index f997f1a..ca77b39 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java @@ -32,7 +32,7 @@ import org.apache.hadoop.io.Text; /** * This iterator dedupes chunks and sets their visibilities to the combined visibility of the refs columns. For example, it would combine - * + * * <pre> * row1 refs uid1\0a A&B V0 * row1 refs uid2\0b C&D V0 @@ -41,88 +41,88 @@ import org.apache.hadoop.io.Text; * row1 ~chunk 0 E&F V1 * row1 ~chunk 0 G&H V1 * </pre> - * + * * into the following - * + * * <pre> * row1 refs uid1\0a A&B V0 * row1 refs uid2\0b C&D V0 * row1 ~chunk 0 (A&B)|(C&D) V1 * </pre> - * + * * {@link VisibilityCombiner} is used to combie the visibilities. */ public class ChunkCombiner implements SortedKeyValueIterator<Key,Value> { - + private SortedKeyValueIterator<Key,Value> source; private SortedKeyValueIterator<Key,Value> refsSource; private static final Collection<ByteSequence> refsColf = Collections.singleton(FileDataIngest.REFS_CF_BS); private Map<Text,byte[]> lastRowVC = Collections.emptyMap(); - + private Key topKey = null; private Value topValue = null; - + public ChunkCombiner() {} - + @Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { this.source = source; this.refsSource = source.deepCopy(env); } - + @Override public boolean hasTop() { return topKey != null; } - + @Override public void next() throws IOException { findTop(); } - + @Override public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException { source.seek(range, columnFamilies, inclusive); findTop(); } - + private void findTop() throws IOException { do { topKey = null; topValue = null; } while (source.hasTop() && _findTop() == null); } - + private byte[] _findTop() throws IOException { long maxTS; - + topKey = new Key(source.getTopKey()); topValue = new Value(source.getTopValue()); source.next(); - + if (!topKey.getColumnFamilyData().equals(FileDataIngest.CHUNK_CF_BS)) return topKey.getColumnVisibility().getBytes(); - + maxTS = topKey.getTimestamp(); - + while (source.hasTop() && source.getTopKey().equals(topKey, PartialKey.ROW_COLFAM_COLQUAL)) { if (source.getTopKey().getTimestamp() > maxTS) maxTS = source.getTopKey().getTimestamp(); - + if (!topValue.equals(source.getTopValue())) throw new RuntimeException("values not equals " + topKey + " " + source.getTopKey() + " : " + diffInfo(topValue, source.getTopValue())); - + source.next(); } - + byte[] vis = getVisFromRefs(); if (vis != null) { topKey = new Key(topKey.getRowData().toArray(), topKey.getColumnFamilyData().toArray(), topKey.getColumnQualifierData().toArray(), vis, maxTS); } return vis; } - + private byte[] getVisFromRefs() throws IOException { Text row = topKey.getRow(); if (lastRowVC.containsKey(row)) @@ -143,34 +143,34 @@ public class ChunkCombiner implements SortedKeyValueIterator<Key,Value> { lastRowVC = Collections.singletonMap(row, vc.get()); return vc.get(); } - + private String diffInfo(Value v1, Value v2) { if (v1.getSize() != v2.getSize()) { return "val len not equal " + v1.getSize() + "!=" + v2.getSize(); } - + byte[] vb1 = v1.get(); byte[] vb2 = v2.get(); - + for (int i = 0; i < vb1.length; i++) { if (vb1[i] != vb2[i]) { return String.format("first diff at offset %,d 0x%02x != 0x%02x", i, 0xff & vb1[i], 0xff & vb2[i]); } } - + return null; } - + @Override public Key getTopKey() { return topKey; } - + @Override public Value getTopValue() { return topValue; } - + @Override public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) { ChunkCombiner cc = new ChunkCombiner(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java index 732b03b..f5da4e5 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java @@ -41,7 +41,7 @@ public class ChunkInputFormat extends InputFormatBase<List<Entry<Key,Value>>,Inp InterruptedException { return new RecordReaderBase<List<Entry<Key,Value>>,InputStream>() { private PeekingIterator<Entry<Key,Value>> peekingScannerIterator; - + @Override public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException { super.initialize(inSplit, attempt); @@ -49,7 +49,7 @@ public class ChunkInputFormat extends InputFormatBase<List<Entry<Key,Value>>,Inp currentK = new ArrayList<Entry<Key,Value>>(); currentV = new ChunkInputStream(); } - + @Override public boolean nextKeyValue() throws IOException, InterruptedException { currentK.clear(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bc67602/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java ---------------------------------------------------------------------- diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java index 30b8521..c902271 100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java @@ -34,26 +34,26 @@ import org.apache.log4j.Logger; */ public class ChunkInputStream extends InputStream { private static final Logger log = Logger.getLogger(ChunkInputStream.class); - + protected PeekingIterator<Entry<Key,Value>> source; protected Key currentKey; protected Set<Text> currentVis; protected int currentChunk; protected int currentChunkSize; protected boolean gotEndMarker; - + protected byte buf[]; protected int count; protected int pos; - + public ChunkInputStream() { source = null; } - + public ChunkInputStream(PeekingIterator<Entry<Key,Value>> in) throws IOException { setSource(in); } - + public void setSource(PeekingIterator<Entry<Key,Value>> in) throws IOException { if (source != null) throw new IOException("setting new source without closing old one"); @@ -65,7 +65,7 @@ public class ChunkInputStream extends InputStream { gotEndMarker = true; return; } - + // read forward until we reach a chunk Entry<Key,Value> entry = source.next(); currentKey = entry.getKey(); @@ -91,7 +91,7 @@ public class ChunkInputStream extends InputStream { throw new IOException("starting chunk number isn't 0 for " + currentKey.getRow()); } } - + private int fill() throws IOException { if (source == null || !source.hasNext()) { if (gotEndMarker) @@ -99,11 +99,11 @@ public class ChunkInputStream extends InputStream { else throw new IOException("no end chunk marker but source has no data"); } - + Entry<Key,Value> entry = source.peek(); Key thisKey = entry.getKey(); log.debug("evaluating key: " + thisKey.toString()); - + // check that we're still on the same row if (!thisKey.equals(currentKey, PartialKey.ROW)) { if (gotEndMarker) @@ -115,39 +115,39 @@ public class ChunkInputStream extends InputStream { } } log.debug("matches current key"); - + // ok to advance the iterator source.next(); - + // check that this is part of a chunk if (!thisKey.getColumnFamily().equals(FileDataIngest.CHUNK_CF)) { log.debug("skipping non-chunk key"); return fill(); } log.debug("is a chunk"); - + // check that the chunk size is the same as the one being read if (currentChunkSize != FileDataIngest.bytesToInt(thisKey.getColumnQualifier().getBytes(), 0)) { log.debug("skipping chunk of different size"); return fill(); } - + // add the visibility to the list if it's not there if (!currentVis.contains(thisKey.getColumnVisibility())) currentVis.add(thisKey.getColumnVisibility()); - + // check to see if it is an identical chunk with a different visibility if (thisKey.getColumnQualifier().equals(currentKey.getColumnQualifier())) { log.debug("skipping identical chunk with different visibility"); return fill(); } - + if (gotEndMarker) { log.debug("got another chunk after end marker: " + currentKey.toString() + " " + thisKey.toString()); clear(); throw new IOException("found extra chunk after end marker"); } - + // got new chunk of the same file, check that it's the next chunk int thisChunk = FileDataIngest.bytesToInt(thisKey.getColumnQualifier().getBytes(), 4); if (thisChunk != currentChunk + 1) { @@ -155,27 +155,27 @@ public class ChunkInputStream extends InputStream { clear(); throw new IOException("missing chunks between " + currentChunk + " and " + thisChunk); } - + currentKey = thisKey; currentChunk = thisChunk; buf = entry.getValue().get(); pos = 0; - + // check to see if it's the last chunk if (buf.length == 0) { gotEndMarker = true; return fill(); } - + return count = buf.length; } - + public Set<Text> getVisibilities() { if (source != null) throw new IllegalStateException("don't get visibilities before chunks have been completely read"); return currentVis; } - + public int read() throws IOException { if (source == null) return -1; @@ -191,7 +191,7 @@ public class ChunkInputStream extends InputStream { } return buf[pos++] & 0xff; } - + @Override public int read(byte[] b, int off, int len) throws IOException { if (b == null) { @@ -201,7 +201,7 @@ public class ChunkInputStream extends InputStream { } else if (len == 0) { return 0; } - + log.debug("filling buffer " + off + " " + len); int total = 0; while (total < len) { @@ -218,7 +218,7 @@ public class ChunkInputStream extends InputStream { } avail = count - pos; } - + int cnt = (avail < len - total) ? avail : len - total; log.debug("copying from local buffer: local pos " + pos + " into pos " + off + " len " + cnt); System.arraycopy(buf, pos, b, off, cnt); @@ -229,7 +229,7 @@ public class ChunkInputStream extends InputStream { log.debug("filled " + total + " bytes"); return total; } - + public void clear() { source = null; buf = null; @@ -237,7 +237,7 @@ public class ChunkInputStream extends InputStream { currentChunk = 0; pos = count = 0; } - + @Override public void close() throws IOException { try {
