ACCUMULO-2503 add formatter tests
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c56ef2ee Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c56ef2ee Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c56ef2ee Branch: refs/heads/master Commit: c56ef2eec1f62e940cb9f137bfd0b2821aa478e7 Parents: d77fd6f Author: Mike Drob <md...@cloudera.com> Authored: Wed Mar 19 19:12:47 2014 -0400 Committer: Mike Drob <md...@cloudera.com> Committed: Tue Mar 25 14:00:08 2014 -0400 ---------------------------------------------------------------------- .../core/util/format/AggregatingFormatter.java | 52 ++++++ .../core/util/format/BinaryFormatter.java | 89 ++------- .../core/util/format/DefaultFormatter.java | 42 +++-- .../core/util/format/DeleterFormatter.java | 3 + .../core/util/format/FormatterFactory.java | 4 + .../accumulo/core/util/format/HexFormatter.java | 8 +- .../ShardedTableDistributionFormatter.java | 16 +- .../util/format/StatisticsDisplayFormatter.java | 16 +- .../util/format/DateStringFormatterTest.java | 61 +++++++ .../core/util/format/DefaultFormatterTest.java | 62 +++++++ .../core/util/format/DeleterFormatterTest.java | 179 +++++++++++++++++++ .../core/util/format/FormatterFactoryTest.java | 45 +++++ .../core/util/format/HexFormatterTest.java | 88 +++++++++ .../ShardedTableDistributionFormatterTest.java | 66 +++++++ .../format/StatisticsDisplayFormatterTest.java | 64 +++++++ core/src/test/resources/log4j.properties | 1 + 16 files changed, 681 insertions(+), 115 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/main/java/org/apache/accumulo/core/util/format/AggregatingFormatter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/format/AggregatingFormatter.java b/core/src/main/java/org/apache/accumulo/core/util/format/AggregatingFormatter.java new file mode 100644 index 0000000..e0791eb --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/util/format/AggregatingFormatter.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.util.format; + +import java.util.Iterator; +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; + +/** + * Formatter that will aggregate entries for various display purposes. + */ +public abstract class AggregatingFormatter extends DefaultFormatter { + @Override + public String next() { + Iterator<Entry<Key,Value>> si = super.getScannerIterator(); + checkState(true); + while (si.hasNext()) + aggregateStats(si.next()); + return getStats(); + } + + /** + * Generate statistics from each {@link Entry}, called for each entry to be iterated over. + * + * @param next + * the next entry to aggregate + */ + protected abstract void aggregateStats(Entry<Key,Value> next); + + /** + * Finalize the aggregation and return the result. Called once at the end. + * + * @return the aggregation results, suitable for printing to the console + */ + protected abstract String getStats(); +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/main/java/org/apache/accumulo/core/util/format/BinaryFormatter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/format/BinaryFormatter.java b/core/src/main/java/org/apache/accumulo/core/util/format/BinaryFormatter.java index 5021d66..b31df18 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/format/BinaryFormatter.java +++ b/core/src/main/java/org/apache/accumulo/core/util/format/BinaryFormatter.java @@ -16,7 +16,6 @@ */ package org.apache.accumulo.core.util.format; -import java.util.Iterator; import java.util.Map.Entry; import org.apache.accumulo.core.data.Key; @@ -24,66 +23,43 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Text; -public class BinaryFormatter implements Formatter { - private Iterator<Entry<Key,Value>> si; - private boolean doTimestamps; +public class BinaryFormatter extends DefaultFormatter { private static int showLength; - @Override - public void initialize(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps) { - checkState(si, false); - si = scanner.iterator(); - doTimestamps = printTimestamps; - } - - public boolean hasNext() { - checkState(si, true); - return si.hasNext(); - } - public String next() { - checkState(si, true); - return formatEntry(si.next(), doTimestamps); - } - - public void remove() { - checkState(si, true); - si.remove(); - } - - static void checkState(Iterator<Entry<Key,Value>> si, boolean expectInitialized) { - if (expectInitialized && si == null) - throw new IllegalStateException("Not initialized"); - if (!expectInitialized && si != null) - throw new IllegalStateException("Already initialized"); + checkState(true); + return formatEntry(getScannerIterator().next(), isDoTimestamps()); } // this should be replaced with something like Record.toString(); + // it would be great if we were able to combine code with DefaultFormatter.formatEntry, but that currently does not respect the showLength option. public static String formatEntry(Entry<Key,Value> entry, boolean showTimestamps) { StringBuilder sb = new StringBuilder(); + Key key = entry.getKey(); + // append row - appendText(sb, entry.getKey().getRow()).append(" "); - + appendText(sb, key.getRow()).append(" "); + // append column family - appendText(sb, entry.getKey().getColumnFamily()).append(":"); - + appendText(sb, key.getColumnFamily()).append(":"); + // append column qualifier - appendText(sb, entry.getKey().getColumnQualifier()).append(" "); - + appendText(sb, key.getColumnQualifier()).append(" "); + // append visibility expression - sb.append(new ColumnVisibility(entry.getKey().getColumnVisibility())); + sb.append(new ColumnVisibility(key.getColumnVisibility())); // append timestamp if (showTimestamps) sb.append(" ").append(entry.getKey().getTimestamp()); // append value - if (entry.getValue() != null && entry.getValue().getSize() > 0) { + Value value = entry.getValue(); + if (value != null && value.getSize() > 0) { sb.append("\t"); - appendValue(sb, entry.getValue()); + appendValue(sb, value); } - return sb.toString(); } @@ -92,41 +68,12 @@ public class BinaryFormatter implements Formatter { } static StringBuilder appendValue(StringBuilder sb, Value value) { - return appendBytes(sb, value.get(), 0, value.get().length); } static StringBuilder appendBytes(StringBuilder sb, byte ba[], int offset, int len) { - if (len > showLength) { - for (int i = 0; i < showLength; i++) { - int c = 0xff & ba[offset + i]; - if (c == '\\') - sb.append("\\\\"); - else if (c >= 32 && c <= 126) - sb.append((char) c); - else - sb.append("\\x").append(String.format("%02X", c)); - } - return sb; - } - - else { - for (int i = 0; i < len; i++) { - - int c = 0xff & ba[offset + i]; - if (c == '\\') - sb.append("\\\\"); - else if (c >= 32 && c <= 126) - sb.append((char) c); - else - sb.append("\\x").append(String.format("%02X", c)); - } - return sb; - } - } - - public Iterator<Entry<Key,Value>> getScannerIterator() { - return si; + int length = Math.min(len, showLength); + return DefaultFormatter.appendBytes(sb, ba, offset, length); } public static void getlength(int length) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/main/java/org/apache/accumulo/core/util/format/DefaultFormatter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/format/DefaultFormatter.java b/core/src/main/java/org/apache/accumulo/core/util/format/DefaultFormatter.java index ee4a220..acd4ba1 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/format/DefaultFormatter.java +++ b/core/src/main/java/org/apache/accumulo/core/util/format/DefaultFormatter.java @@ -56,13 +56,13 @@ public class DefaultFormatter implements Formatter { @Override public void initialize(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps) { - checkState(si, false); + checkState(false); si = scanner.iterator(); doTimestamps = printTimestamps; } public boolean hasNext() { - checkState(si, true); + checkState(true); return si.hasNext(); } @@ -77,22 +77,22 @@ public class DefaultFormatter implements Formatter { } protected String next(DateFormat timestampFormat) { - checkState(si, true); + checkState(true); return formatEntry(si.next(), timestampFormat); } public void remove() { - checkState(si, true); + checkState(true); si.remove(); } - static void checkState(Iterator<Entry<Key,Value>> si, boolean expectInitialized) { + protected void checkState(boolean expectInitialized) { if (expectInitialized && si == null) throw new IllegalStateException("Not initialized"); if (!expectInitialized && si != null) throw new IllegalStateException("Already initialized"); } - + // this should be replaced with something like Record.toString(); public static String formatEntry(Entry<Key,Value> entry, boolean showTimestamps) { DateFormat timestampFormat = null; @@ -106,6 +106,7 @@ public class DefaultFormatter implements Formatter { /* so a new date object doesn't get created for every record in the scan result */ private static ThreadLocal<Date> tmpDate = new ThreadLocal<Date>() { + @Override protected Date initialValue() { return new Date(); } @@ -114,32 +115,37 @@ public class DefaultFormatter implements Formatter { public static String formatEntry(Entry<Key,Value> entry, DateFormat timestampFormat) { StringBuilder sb = new StringBuilder(); + Key key = entry.getKey(); + // append row - appendText(sb, entry.getKey().getRow()).append(" "); - + appendText(sb, key.getRow()).append(" "); + // append column family - appendText(sb, entry.getKey().getColumnFamily()).append(":"); - + appendText(sb, key.getColumnFamily()).append(":"); + // append column qualifier - appendText(sb, entry.getKey().getColumnQualifier()).append(" "); - + appendText(sb, key.getColumnQualifier()).append(" "); + // append visibility expression - sb.append(new ColumnVisibility(entry.getKey().getColumnVisibility())); + sb.append(new ColumnVisibility(key.getColumnVisibility())); // append timestamp if (timestampFormat != null) { tmpDate.get().setTime(entry.getKey().getTimestamp()); sb.append(" ").append(timestampFormat.format(tmpDate.get())); } + + Value value = entry.getValue(); + // append value - if (entry.getValue() != null && entry.getValue().getSize() > 0) { + if (value != null && value.getSize() > 0) { sb.append("\t"); - appendValue(sb, entry.getValue()); + appendValue(sb, value); } return sb.toString(); } - + static StringBuilder appendText(StringBuilder sb, Text t) { return appendBytes(sb, t.getBytes(), 0, t.getLength()); } @@ -164,4 +170,8 @@ public class DefaultFormatter implements Formatter { public Iterator<Entry<Key,Value>> getScannerIterator() { return si; } + + protected boolean isDoTimestamps() { + return doTimestamps; + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/main/java/org/apache/accumulo/core/util/format/DeleterFormatter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/format/DeleterFormatter.java b/core/src/main/java/org/apache/accumulo/core/util/format/DeleterFormatter.java index 8547f12..7ac0510 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/format/DeleterFormatter.java +++ b/core/src/main/java/org/apache/accumulo/core/util/format/DeleterFormatter.java @@ -63,6 +63,9 @@ public class DeleterFormatter extends DefaultFormatter { return true; } + /** + * @return null, because the iteration will provide prompts and handle deletes internally. + */ @Override public String next() { Entry<Key,Value> next = getScannerIterator().next(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/main/java/org/apache/accumulo/core/util/format/FormatterFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/format/FormatterFactory.java b/core/src/main/java/org/apache/accumulo/core/util/format/FormatterFactory.java index 5451843..27299ee 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/format/FormatterFactory.java +++ b/core/src/main/java/org/apache/accumulo/core/util/format/FormatterFactory.java @@ -40,4 +40,8 @@ public class FormatterFactory { public static Formatter getDefaultFormatter(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps) { return getFormatter(DefaultFormatter.class, scanner, printTimestamps); } + + private FormatterFactory() { + // prevent instantiation + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java b/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java index 1def712..b636278 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java +++ b/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java @@ -116,21 +116,21 @@ public class HexFormatter implements Formatter, ScanInterpreter { @Override public Text interpretBeginRow(Text row) { - return new Text(toBinary(row.toString())); + return interpretRow(row); } @Override public Text interpretEndRow(Text row) { - return new Text(toBinary(row.toString())); + return interpretRow(row); } @Override public Text interpretColumnFamily(Text cf) { - return new Text(toBinary(cf.toString())); + return interpretRow(cf); } @Override public Text interpretColumnQualifier(Text cq) { - return new Text(toBinary(cq.toString())); + return interpretRow(cq); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/main/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatter.java b/core/src/main/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatter.java index 577167a..f81209f 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatter.java +++ b/core/src/main/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatter.java @@ -18,7 +18,6 @@ package org.apache.accumulo.core.util.format; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; @@ -33,20 +32,12 @@ import org.apache.accumulo.core.data.Value; * * scan -b tableId -c ~tab:loc */ -public class ShardedTableDistributionFormatter extends DefaultFormatter { +public class ShardedTableDistributionFormatter extends AggregatingFormatter { private Map<String,HashSet<String>> countsByDay = new HashMap<String,HashSet<String>>(); @Override - public String next() { - Iterator<Entry<Key,Value>> si = super.getScannerIterator(); - checkState(si, true); - while (si.hasNext()) - aggregateStats(si.next()); - return getStats(); - } - - private void aggregateStats(Entry<Key,Value> entry) { + protected void aggregateStats(Entry<Key,Value> entry) { if (entry.getKey().getColumnFamily().toString().equals("~tab") && entry.getKey().getColumnQualifier().toString().equals("loc")) { // The row for the sharded table should look like: <tableId>;yyyyMMhh_N String row = entry.getKey().getRow().toString(); @@ -65,7 +56,8 @@ public class ShardedTableDistributionFormatter extends DefaultFormatter { } } - private String getStats() { + @Override + protected String getStats() { StringBuilder buf = new StringBuilder(); buf.append("DAY \t\tSERVERS\n"); buf.append("------\t\t-------\n"); http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/main/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatter.java b/core/src/main/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatter.java index dd9de6c..98d4d28 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatter.java +++ b/core/src/main/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatter.java @@ -17,7 +17,6 @@ package org.apache.accumulo.core.util.format; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; @@ -28,22 +27,14 @@ import org.apache.accumulo.core.data.Value; * Does not show contents from scan, only displays statistics. Beware that this work is being done client side and this was developed as a utility for * debugging. If used on large result sets it will likely fail. */ -public class StatisticsDisplayFormatter extends DefaultFormatter { +public class StatisticsDisplayFormatter extends AggregatingFormatter { private Map<String,Long> classifications = new HashMap<String,Long>(); private Map<String,Long> columnFamilies = new HashMap<String,Long>(); private Map<String,Long> columnQualifiers = new HashMap<String,Long>(); private long total = 0; @Override - public String next() { - Iterator<Entry<Key,Value>> si = super.getScannerIterator(); - checkState(si, true); - while (si.hasNext()) - aggregateStats(si.next()); - return getStats(); - } - - private void aggregateStats(Entry<Key,Value> entry) { + protected void aggregateStats(Entry<Key,Value> entry) { String key; Long count; @@ -62,7 +53,8 @@ public class StatisticsDisplayFormatter extends DefaultFormatter { ++total; } - private String getStats() { + @Override + protected String getStats() { StringBuilder buf = new StringBuilder(); buf.append("CLASSIFICATIONS:\n"); buf.append("----------------\n"); http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/test/java/org/apache/accumulo/core/util/format/DateStringFormatterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/DateStringFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/DateStringFormatterTest.java new file mode 100644 index 0000000..69b2e11 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/util/format/DateStringFormatterTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.util.format; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.TreeMap; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.junit.Before; +import org.junit.Test; + +public class DateStringFormatterTest { + DateStringFormatter formatter; + + Map<Key,Value> data; + + @Before + public void setUp() { + formatter = new DateStringFormatter(); + data = new TreeMap<Key,Value>(); + data.put(new Key("", "", "", 0), new Value()); + } + + @Test + public void testTimestamps() { + formatter.initialize(data.entrySet(), true); + + assertTrue(formatter.hasNext()); + assertTrue(formatter.next().endsWith("1969/12/31 19:00:00.000")); + } + + @Test + public void testNoTimestamps() { + data.put(new Key("", "", "", 1), new Value()); + + assertEquals(2, data.size()); + + formatter.initialize(data.entrySet(), false); + + assertEquals(formatter.next(), formatter.next()); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java new file mode 100644 index 0000000..dd4d93f --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/util/format/DefaultFormatterTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.util.format; + +import static org.junit.Assert.assertEquals; + +import java.util.Collections; +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.hadoop.io.Text; +import org.junit.Before; +import org.junit.Test; + +public class DefaultFormatterTest { + + DefaultFormatter df; + Iterable<Entry<Key,Value>> empty = Collections.<Key,Value> emptyMap().entrySet(); + + @Before + public void setUp() { + df = new DefaultFormatter(); + } + + @Test(expected = IllegalStateException.class) + public void testDoubleInitialize() { + df.initialize(empty, true); + df.initialize(empty, true); + } + + @Test(expected = IllegalStateException.class) + public void testNextBeforeInitialize() { + df.hasNext(); + } + + @Test + public void testAppendBytes() { + StringBuilder sb = new StringBuilder(); + byte[] data = new byte[] { 0, '\\', 'x', -0x01 }; + + DefaultFormatter.appendValue(sb, new Value()); + assertEquals("", sb.toString()); + + DefaultFormatter.appendText(sb, new Text(data)); + assertEquals("\\x00\\\\x\\xFF", sb.toString()); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java new file mode 100644 index 0000000..4daf676 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.util.format; + +import static org.apache.accumulo.core.Constants.UTF8; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.Map; +import java.util.TreeMap; + +import jline.console.ConsoleReader; + +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.util.shell.Shell; +import org.junit.Before; +import org.junit.Test; + +public class DeleterFormatterTest { + DeleterFormatter formatter; + Map<Key,Value> data; + BatchWriter writer; + BatchWriter exceptionWriter; + Shell shellState; + + ByteArrayOutputStream baos; + ConsoleReader reader; + + SettableInputStream input; + + class SettableInputStream extends InputStream { + ByteArrayInputStream bais; + + @Override + public int read() throws IOException { + return bais.read(); + } + + public void set(String in) { + bais = new ByteArrayInputStream(in.getBytes(UTF8)); + } + + public void set(byte... in) { + bais = new ByteArrayInputStream(in); + } + }; + + @Before + public void setUp() throws IOException, MutationsRejectedException { + input = new SettableInputStream(); + baos = new ByteArrayOutputStream(); + + MutationsRejectedException mre = createMock(MutationsRejectedException.class); + + writer = createNiceMock(BatchWriter.class); + exceptionWriter = createNiceMock(BatchWriter.class); + exceptionWriter.close(); + expectLastCall().andThrow(mre); + exceptionWriter.addMutation(anyObject(Mutation.class)); + expectLastCall().andThrow(mre); + + shellState = createNiceMock(Shell.class); + + reader = new ConsoleReader(input, baos); + expect(shellState.getReader()).andReturn(reader).anyTimes(); + + replay(writer, exceptionWriter, shellState); + + data = new TreeMap<Key,Value>(); + data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF8))); + } + + @Test + public void testEmpty() { + formatter = new DeleterFormatter(writer, Collections.<Key,Value> emptyMap().entrySet(), true, shellState, true); + assertFalse(formatter.hasNext()); + } + + @Test + public void testSingle() throws IOException { + formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, true); + + assertTrue(formatter.hasNext()); + assertNull(formatter.next()); + + verify("[DELETED]", " r ", "cf", "cq", "value"); + } + + @Test + public void testNo() throws IOException { + input.set("no\n"); + data.put(new Key("z"), new Value("v2".getBytes(UTF8))); + formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false); + + assertTrue(formatter.hasNext()); + assertNull(formatter.next()); + + verify("[SKIPPED]", " r ", "cf", "cq", "value"); + + assertTrue(formatter.hasNext()); + } + + @Test + public void testNoConfirmation() throws IOException { + input.set((byte) -1); + data.put(new Key("z"), new Value("v2".getBytes(UTF8))); + formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false); + + assertTrue(formatter.hasNext()); + assertNull(formatter.next()); + + verify("[SKIPPED]", " r ", "cf", "cq", "value"); + + assertFalse(formatter.hasNext()); + } + + @Test + public void testYes() throws IOException { + input.set("y\nyes\n"); + data.put(new Key("z"), new Value("v2".getBytes(UTF8))); + formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false); + + assertTrue(formatter.hasNext()); + assertNull(formatter.next()); + verify("[DELETED]", " r ", "cf", "cq", "value"); + + assertTrue(formatter.hasNext()); + assertNull(formatter.next()); + verify("[DELETED]", " z ", "v2"); + } + + @Test + public void testMutationException() { + formatter = new DeleterFormatter(exceptionWriter, data.entrySet(), true, shellState, true); + + assertTrue(formatter.hasNext()); + assertNull(formatter.next()); + assertFalse(formatter.hasNext()); + } + + private void verify(String... chunks) throws IOException { + reader.flush(); + + String output = baos.toString(); + for (String chunk : chunks) { + assertTrue(output.contains(chunk)); + } + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/test/java/org/apache/accumulo/core/util/format/FormatterFactoryTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/FormatterFactoryTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/FormatterFactoryTest.java new file mode 100644 index 0000000..d379dee --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/util/format/FormatterFactoryTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.util.format; + +import static org.junit.Assert.assertEquals; + +import java.util.Collections; +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.junit.Before; +import org.junit.Test; + +public class FormatterFactoryTest { + + Iterable<Entry<Key,Value>> scanner; + + @Before + public void setUp() { + scanner = Collections.<Key,Value> emptyMap().entrySet(); + } + + @Test + public void testGetDefaultFormatter() { + Formatter defaultFormatter = FormatterFactory.getDefaultFormatter(scanner, true); + Formatter bogusFormatter = FormatterFactory.getFormatter(Formatter.class, scanner, true); + assertEquals(defaultFormatter.getClass(), bogusFormatter.getClass()); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java new file mode 100644 index 0000000..7f4f3e5 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/util/format/HexFormatterTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.util.format; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.TreeMap; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.hadoop.io.Text; +import org.junit.Before; +import org.junit.Test; + +public class HexFormatterTest { + HexFormatter formatter; + + Map<Key,Value> data; + + @Before + public void setUp() { + data = new TreeMap<Key,Value>(); + formatter = new HexFormatter(); + } + + @Test + public void testInitialize() { + data.put(new Key(), new Value()); + formatter.initialize(data.entrySet(), false); + + assertTrue(formatter.hasNext()); + assertEquals(" " + " " + " [" + "] ", formatter.next()); + } + + @Test + public void testInterpretRow() { + assertEquals(new Text(), formatter.interpretRow(new Text())); + assertEquals(new Text("\0"), formatter.interpretRow(new Text("0"))); + } + + + @Test + public void testRoundTripRows() { + Text bytes = new Text(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}); + data.put(new Key(bytes), new Value()); + + formatter.initialize(data.entrySet(), false); + + String row = formatter.next().split(" ")[0]; + assertEquals("0001-0203-0405-0607-0809-0a0b-0c0d-0e0f", row); + assertEquals(bytes, formatter.interpretRow(new Text(row))); + } + + @Test(expected = IllegalArgumentException.class) + public void testInterpretBadRow0() { + formatter.interpretRow(new Text("!")); + } + + @Test(expected = IllegalArgumentException.class) + public void testInterpretBadRow1() { + formatter.interpretRow(new Text("z")); + } + + @Test + public void testTimestamps() { + long now = System.currentTimeMillis(); + data.put(new Key("", "", "", now), new Value()); + formatter.initialize(data.entrySet(), true); + String entry = formatter.next().split("\\s+")[2]; + assertEquals(now, Long.valueOf(entry).longValue()); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java new file mode 100644 index 0000000..1707dda --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.util.format; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.TreeMap; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.junit.Before; +import org.junit.Test; + +public class ShardedTableDistributionFormatterTest { + ShardedTableDistributionFormatter formatter; + + Map<Key,Value> data; + + @Before + public void setUp() { + data = new TreeMap<Key,Value>(); + formatter = new ShardedTableDistributionFormatter(); + } + + @Test + public void testInitialize() { + data.put(new Key(), new Value()); + data.put(new Key("r", "~tab"), new Value()); + formatter.initialize(data.entrySet(), false); + + assertTrue(formatter.hasNext()); + formatter.next(); + assertFalse(formatter.hasNext()); + } + + @Test + public void testAggregate() { + data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(Constants.UTF8))); + data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(Constants.UTF8))); + data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(Constants.UTF8))); + + formatter.initialize(data.entrySet(), false); + + String[] result = formatter.next().split("\n"); + assertTrue(result[2].endsWith("\t1")); + assertTrue(result[3].endsWith("\t2")); + assertFalse(formatter.hasNext()); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/test/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatterTest.java new file mode 100644 index 0000000..93c948c --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/util/format/StatisticsDisplayFormatterTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.util.format; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.TreeMap; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.junit.Before; +import org.junit.Test; + +public class StatisticsDisplayFormatterTest { + StatisticsDisplayFormatter formatter; + + Map<Key,Value> data; + + @Before + public void setUp() { + data = new TreeMap<Key,Value>(); + formatter = new StatisticsDisplayFormatter(); + } + + @Test + public void testInitialize() { + data.put(new Key(), new Value()); + formatter.initialize(data.entrySet(), false); + + assertTrue(formatter.hasNext()); + } + + @Test + public void testAggregate() { + data.put(new Key("", "", "", 1), new Value()); + data.put(new Key("", "", "", 2), new Value()); + formatter.initialize(data.entrySet(), false); + + String[] output = formatter.next().split("\n"); + assertTrue(output[2].endsWith(": 1")); + assertTrue(output[5].endsWith(": 1")); + assertTrue(output[8].endsWith(": 1")); + assertEquals("2 entries matched.", output[9]); + + assertFalse(formatter.hasNext()); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c56ef2ee/core/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index dfc93bf..9f968f8 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -25,3 +25,4 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN log4j.logger.org.apache.hadoop.mapred=ERROR log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR +log4j.logger.org.apache.accumulo.core.util.format=FATAL
