ACCUMULO-1854 Update tests for hadoop-2 compatibility
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c88d87aa Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c88d87aa Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c88d87aa Branch: refs/heads/master Commit: c88d87aa08819ed94fa9aae8583985b4fcf8e588 Parents: fae8282 Author: Josh Elser <els...@apache.org> Authored: Sat Nov 23 13:49:14 2013 -0500 Committer: Josh Elser <els...@apache.org> Committed: Sat Nov 23 13:49:14 2013 -0500 ---------------------------------------------------------------------- .../mapreduce/AccumuloInputFormatTest.java | 198 ++++++++----------- .../BadPasswordSplitsAccumuloInputFormat.java | 26 +++ .../EmptySplitsAccumuloInputFormat.java | 21 ++ 3 files changed, 129 insertions(+), 116 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java index f9ccdf1..93dba65 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -46,8 +47,10 @@ import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -60,11 +63,9 @@ import org.junit.Assert; import org.junit.Test; public class AccumuloInputFormatTest { - + private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName(); - private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance"; - private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1"; - + /** * Test basic setting & getting of max versions. * @@ -79,7 +80,7 @@ public class AccumuloInputFormatTest { int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration()); assertEquals(1, version); } - + /** * Test max versions with an invalid value. * @@ -92,7 +93,7 @@ public class AccumuloInputFormatTest { Job job = new Job(); AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0); } - + /** * Test no max version configured. * @@ -104,7 +105,7 @@ public class AccumuloInputFormatTest { Job job = new Job(); assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration())); } - + /** * Check that the iterator configuration is getting stored in the Job conf correctly. * @@ -113,7 +114,7 @@ public class AccumuloInputFormatTest { @Test public void testSetIterator() throws IOException { Job job = new Job(); - + IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"); AccumuloInputFormat.addIterator(job, is); Configuration conf = job.getConfiguration(); @@ -122,36 +123,36 @@ public class AccumuloInputFormatTest { String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators"); assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators); } - + @Test public void testAddIterator() throws IOException { Job job = new Job(); - + AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class)); AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator")); IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"); iter.addOption("v1", "1"); iter.addOption("junk", "\0omg:!\\xyzzy"); AccumuloInputFormat.addIterator(job, iter); - + List<IteratorSetting> list = AccumuloInputFormat.getIterators(job); - + // Check the list size assertTrue(list.size() == 3); - + // Walk the list and make sure our settings are correct IteratorSetting setting = list.get(0); assertEquals(1, setting.getPriority()); assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass()); assertEquals("WholeRow", setting.getName()); assertEquals(0, setting.getOptions().size()); - + setting = list.get(1); assertEquals(2, setting.getPriority()); assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass()); assertEquals("Versions", setting.getName()); assertEquals(0, setting.getOptions().size()); - + setting = list.get(2); assertEquals(3, setting.getPriority()); assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass()); @@ -160,7 +161,7 @@ public class AccumuloInputFormatTest { assertEquals("1", setting.getOptions().get("v1")); assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk")); } - + /** * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There * should be no exceptions thrown when trying to parse these types of option entries. @@ -175,12 +176,12 @@ public class AccumuloInputFormatTest { someSetting.addOption(key, value); Job job = new Job(); AccumuloInputFormat.addIterator(job, someSetting); - + List<IteratorSetting> list = AccumuloInputFormat.getIterators(job); assertEquals(1, list.size()); assertEquals(1, list.get(0).getOptions().size()); assertEquals(list.get(0).getOptions().get(key), value); - + someSetting.addOption(key + "2", value); someSetting.setPriority(2); someSetting.setName("it2"); @@ -193,7 +194,7 @@ public class AccumuloInputFormatTest { assertEquals(list.get(1).getOptions().get(key), value); assertEquals(list.get(1).getOptions().get(key + "2"), value); } - + /** * Test getting iterator settings for multiple iterators set * @@ -202,55 +203,55 @@ public class AccumuloInputFormatTest { @Test public void testGetIteratorSettings() throws IOException { Job job = new Job(); - + AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator")); AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator")); AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator")); - + List<IteratorSetting> list = AccumuloInputFormat.getIterators(job); - + // Check the list size assertTrue(list.size() == 3); - + // Walk the list and make sure our settings are correct IteratorSetting setting = list.get(0); assertEquals(1, setting.getPriority()); assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass()); assertEquals("WholeRow", setting.getName()); - + setting = list.get(1); assertEquals(2, setting.getPriority()); assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass()); assertEquals("Versions", setting.getName()); - + setting = list.get(2); assertEquals(3, setting.getPriority()); assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass()); assertEquals("Count", setting.getName()); - + } - + @Test public void testSetRegex() throws IOException { Job job = new Job(); - + String regex = ">\"*%<>\'\\"; - + IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class); RegExFilter.setRegexs(is, regex, null, null, null, false); AccumuloInputFormat.addIterator(job, is); - + assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName())); } - + private static AssertionError e1 = null; private static AssertionError e2 = null; - + private static class MRTester extends Configured implements Tool { private static class TestMapper extends Mapper<Key,Value,Key,Value> { Key key = null; int count = 0; - + @Override protected void map(Key k, Value v, Context context) throws IOException, InterruptedException { try { @@ -264,7 +265,7 @@ public class AccumuloInputFormatTest { key = new Key(k); count++; } - + @Override protected void cleanup(Context context) throws IOException, InterruptedException { try { @@ -274,46 +275,52 @@ public class AccumuloInputFormatTest { } } } - + @Override public int run(String[] args) throws Exception { - - if (args.length != 3) { - throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>"); + + if (args.length != 5) { + throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table> <instanceName> <inputFormatClass>"); } - + String user = args[0]; String pass = args[1]; String table = args[2]; - + String instanceName = args[3]; + String inputFormatClassName = args[4]; + Class<? extends InputFormat> inputFormatClass = (Class<? extends InputFormat>) Class.forName(inputFormatClassName); + Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); - - job.setInputFormatClass(AccumuloInputFormat.class); - + + job.setInputFormatClass(inputFormatClass); + AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass)); AccumuloInputFormat.setInputTableName(job, table); - AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME); - + AccumuloInputFormat.setMockInstance(job, instanceName); + job.setMapperClass(TestMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormatClass(NullOutputFormat.class); - + job.setNumReduceTasks(0); - + job.waitForCompletion(true); - + return job.isSuccessful() ? 0 : 1; } - - public static void main(String[] args) throws Exception { - assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args)); + + public static int main(String[] args) throws Exception { + return ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args); } } - + @Test public void testMap() throws Exception { + final String INSTANCE_NAME = PREFIX + "_mapreduce_instance"; + final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1"; + MockInstance mockInstance = new MockInstance(INSTANCE_NAME); Connector c = mockInstance.getConnector("root", new PasswordToken("")); c.tableOperations().create(TEST_TABLE_1); @@ -324,13 +331,12 @@ public class AccumuloInputFormatTest { bw.addMutation(m); } bw.close(); - - MRTester.main(new String[] {"root", "", TEST_TABLE_1}); + + Assert.assertEquals(0, MRTester.main(new String[] {"root", "", TEST_TABLE_1, INSTANCE_NAME, AccumuloInputFormat.class.getCanonicalName()})); assertNull(e1); assertNull(e2); } - - @SuppressWarnings("deprecation") + @Test public void testCorrectRangeInputSplits() throws Exception { Job job = new Job(new Configuration(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); @@ -354,19 +360,19 @@ public class AccumuloInputFormatTest { AccumuloInputFormat.setLocalIterators(job, localIters); AccumuloInputFormat.fetchColumns(job, fetchColumns); AccumuloInputFormat.setLogLevel(job, level); - + AccumuloInputFormat aif = new AccumuloInputFormat(); - + List<InputSplit> splits = aif.getSplits(job); - + Assert.assertEquals(1, splits.size()); - + InputSplit split = splits.get(0); - + Assert.assertEquals(RangeInputSplit.class, split.getClass()); - + RangeInputSplit risplit = (RangeInputSplit) split; - + Assert.assertEquals(username, risplit.getPrincipal()); Assert.assertEquals(table, risplit.getTable()); Assert.assertEquals(password, risplit.getToken()); @@ -377,7 +383,7 @@ public class AccumuloInputFormatTest { Assert.assertEquals(fetchColumns, risplit.getFetchedColumns()); Assert.assertEquals(level, risplit.getLogLevel()); } - + static class TestMapper extends Mapper<Key,Value,Key,Value> { Key key = null; int count = 0; @@ -397,7 +403,7 @@ public class AccumuloInputFormatTest { public void testPartialInputSplitDelegationToConfiguration() throws Exception { String user = "testPartialInputSplitUser"; PasswordToken password = new PasswordToken(""); - + MockInstance mockInstance = new MockInstance("testPartialInputSplitDelegationToConfiguration"); Connector c = mockInstance.getConnector(user, password); c.tableOperations().create("testtable"); @@ -409,37 +415,17 @@ public class AccumuloInputFormatTest { } bw.close(); - Job job = new Job(new Configuration()); - job.setInputFormatClass(AccumuloInputFormat.class); - job.setMapperClass(TestMapper.class); - job.setNumReduceTasks(0); - AccumuloInputFormat.setConnectorInfo(job, user, password); - AccumuloInputFormat.setInputTableName(job, "testtable"); - AccumuloInputFormat.setScanAuthorizations(job, new Authorizations()); - AccumuloInputFormat.setMockInstance(job, "testPartialInputSplitDelegationToConfiguration"); - - AccumuloInputFormat input = new AccumuloInputFormat(); - List<InputSplit> splits = input.getSplits(job); - assertEquals(splits.size(), 1); - - TestMapper mapper = (TestMapper) job.getMapperClass().newInstance(); - - RangeInputSplit emptySplit = new RangeInputSplit(); - - // Using an empty split should fall back to the information in the Job's Configuration - TaskAttemptID id = new TaskAttemptID(); - TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id); - RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, attempt); - Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), id, reader, null, null, null, emptySplit); - reader.initialize(emptySplit, context); - mapper.run(context); + Assert.assertEquals(0, MRTester.main(new String[] {user, "", "testtable", "testPartialInputSplitDelegationToConfiguration", + EmptySplitsAccumuloInputFormat.class.getCanonicalName()})); + assertNull(e1); + assertNull(e2); } - @Test(expected = IOException.class) + @Test public void testPartialFailedInputSplitDelegationToConfiguration() throws Exception { String user = "testPartialFailedInputSplit"; PasswordToken password = new PasswordToken(""); - + MockInstance mockInstance = new MockInstance("testPartialFailedInputSplitDelegationToConfiguration"); Connector c = mockInstance.getConnector(user, password); c.tableOperations().create("testtable"); @@ -451,30 +437,10 @@ public class AccumuloInputFormatTest { } bw.close(); - Job job = new Job(new Configuration()); - job.setInputFormatClass(AccumuloInputFormat.class); - job.setMapperClass(TestMapper.class); - job.setNumReduceTasks(0); - AccumuloInputFormat.setConnectorInfo(job, user, password); - AccumuloInputFormat.setInputTableName(job, "testtable"); - AccumuloInputFormat.setMockInstance(job, "testPartialFailedInputSplitDelegationToConfiguration"); - - AccumuloInputFormat input = new AccumuloInputFormat(); - List<InputSplit> splits = input.getSplits(job); - assertEquals(splits.size(), 1); - - TestMapper mapper = (TestMapper) job.getMapperClass().newInstance(); - - RangeInputSplit emptySplit = new RangeInputSplit(); - emptySplit.setPrincipal("root"); - emptySplit.setToken(new PasswordToken("anythingelse")); - - // Using an empty split should fall back to the information in the Job's Configuration - TaskAttemptID id = new TaskAttemptID(); - TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id); - RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, attempt); - Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), id, reader, null, null, null, emptySplit); - reader.initialize(emptySplit, context); - mapper.run(context); + // We should fail before we even get into the Mapper because we can't make the RecordReader + Assert.assertEquals(1, MRTester.main(new String[] {user, "", "testtable", "testPartialFailedInputSplitDelegationToConfiguration", + BadPasswordSplitsAccumuloInputFormat.class.getCanonicalName()})); + assertNull(e1); + assertNull(e2); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java new file mode 100644 index 0000000..846208b --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java @@ -0,0 +1,26 @@ +package org.apache.accumulo.core.client.mapreduce; + +import java.io.IOException; +import java.util.List; + +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; + +/** + * AccumuloInputFormat which returns an "empty" RangeInputSplit + */ +public class BadPasswordSplitsAccumuloInputFormat extends AccumuloInputFormat { + + @Override + public List<InputSplit> getSplits(JobContext context) throws IOException { + List<InputSplit> splits = super.getSplits(context); + + for (InputSplit split : splits) { + RangeInputSplit rangeSplit = (RangeInputSplit) split; + rangeSplit.setToken(new PasswordToken("anythingelse")); + } + + return splits; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java new file mode 100644 index 0000000..91db378 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java @@ -0,0 +1,21 @@ +package org.apache.accumulo.core.client.mapreduce; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; + +/** + * AccumuloInputFormat which returns an "empty" RangeInputSplit + */ +public class EmptySplitsAccumuloInputFormat extends AccumuloInputFormat { + + @Override + public List<InputSplit> getSplits(JobContext context) throws IOException { + super.getSplits(context); + + return Arrays.<InputSplit> asList(new RangeInputSplit()); + } +} \ No newline at end of file
