ACCUMULO-2962 (De)serialize IteratorSettings on RangeInputSplit(s). Update the unit tests as they accidentally omitted the iterator settings.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/023be574 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/023be574 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/023be574 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 023be574803a7aa88ce316a9f499c1e29bc61623 Parents: 75c053d Author: Josh Elser <els...@apache.org> Authored: Mon Jun 30 23:44:32 2014 -0400 Committer: Josh Elser <els...@apache.org> Committed: Mon Jul 7 14:42:22 2014 -0400 ---------------------------------------------------------------------- .../core/client/mapreduce/RangeInputSplit.java | 16 ++++++++++++++++ .../core/client/mapred/RangeInputSplitTest.java | 16 ++++++++++++++++ .../core/client/mapreduce/RangeInputSplitTest.java | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/023be574/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java index 73c9b59..05316a1 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java @@ -205,6 +205,14 @@ public class RangeInputSplit extends InputSplit implements Writable { } if (in.readBoolean()) { + int numIterators = in.readInt(); + iterators = new ArrayList<IteratorSetting>(numIterators); + for (int i = 0; i < numIterators; i++) { + iterators.add(new IteratorSetting(in)); + } + } + + if (in.readBoolean()) { level = Level.toLevel(in.readInt()); } } @@ -275,6 +283,14 @@ public class RangeInputSplit extends InputSplit implements Writable { out.writeUTF(zooKeepers); } + out.writeBoolean(null != iterators); + if (null != iterators) { + out.writeInt(iterators.size()); + for (IteratorSetting iterator : iterators) { + iterator.write(out); + } + } + out.writeBoolean(null != level); if (null != level) { out.writeInt(level.toInt()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/023be574/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java index 590f2b5..6e4e30e 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java @@ -21,13 +21,17 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.iterators.user.SummingCombiner; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.Pair; import org.apache.hadoop.io.Text; @@ -63,6 +67,16 @@ public class RangeInputSplitTest { fetchedColumns.add(new Pair<Text,Text>(new Text("colf1"), new Text("colq1"))); fetchedColumns.add(new Pair<Text,Text>(new Text("colf2"), new Text("colq2"))); + + // Fake some iterators + ArrayList<IteratorSetting> iterators = new ArrayList<IteratorSetting>(); + IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class); + setting.addOption("foo", "bar"); + iterators.add(setting); + + setting = new IteratorSetting(100, WholeRowIterator.class); + setting.addOption("bar", "foo"); + iterators.add(setting); split.setAuths(new Authorizations("foo")); split.setOffline(true); @@ -74,6 +88,7 @@ public class RangeInputSplitTest { split.setInstanceName("instance"); split.setMockInstance(true); split.setZooKeepers("localhost"); + split.setIterators(iterators); split.setLogLevel(Level.WARN); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -99,6 +114,7 @@ public class RangeInputSplitTest { Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName()); Assert.assertEquals(split.isMockInstance(), newSplit.isMockInstance()); Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers()); + Assert.assertEquals(split.getIterators(), newSplit.getIterators()); Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel()); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/023be574/core/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java index 62ea27b..78597ac 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java @@ -21,13 +21,17 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.iterators.user.SummingCombiner; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.Pair; import org.apache.hadoop.io.Text; @@ -63,6 +67,16 @@ public class RangeInputSplitTest { fetchedColumns.add(new Pair<Text,Text>(new Text("colf1"), new Text("colq1"))); fetchedColumns.add(new Pair<Text,Text>(new Text("colf2"), new Text("colq2"))); + + // Fake some iterators + ArrayList<IteratorSetting> iterators = new ArrayList<IteratorSetting>(); + IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class); + setting.addOption("foo", "bar"); + iterators.add(setting); + + setting = new IteratorSetting(100, WholeRowIterator.class); + setting.addOption("bar", "foo"); + iterators.add(setting); split.setAuths(new Authorizations("foo")); split.setOffline(true); @@ -74,6 +88,7 @@ public class RangeInputSplitTest { split.setInstanceName("instance"); split.setMockInstance(true); split.setZooKeepers("localhost"); + split.setIterators(iterators); split.setLogLevel(Level.WARN); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -99,6 +114,7 @@ public class RangeInputSplitTest { Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName()); Assert.assertEquals(split.isMockInstance(), newSplit.isMockInstance()); Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers()); + Assert.assertEquals(split.getIterators(), newSplit.getIterators()); Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel()); }
