ACCUMULO-1783 Update this repo with the upstream changes I've been pushing Pig's way.
Project: http://git-wip-us.apache.org/repos/asf/accumulo-pig/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo-pig/commit/6820d50a Tree: http://git-wip-us.apache.org/repos/asf/accumulo-pig/tree/6820d50a Diff: http://git-wip-us.apache.org/repos/asf/accumulo-pig/diff/6820d50a Branch: refs/heads/ACCUMULO-1783-1.5 Commit: 6820d50ae74be9e3bdabfd82cadc9c17ddef41c2 Parents: 144f19e Author: Josh Elser <els...@apache.org> Authored: Mon Dec 30 21:17:13 2013 -0500 Committer: Josh Elser <els...@apache.org> Committed: Mon Dec 30 21:17:13 2013 -0500 ---------------------------------------------------------------------- .gitignore | 1 + pom.xml | 4 +- .../accumulo/pig/AbstractAccumuloStorage.java | 26 +++- .../apache/accumulo/pig/AccumuloStorage.java | 16 ++ .../java/org/apache/accumulo/pig/FORMAT.java | 16 ++ .../accumulo/pig/AccumuloPigClusterTest.java | 16 ++ .../pig/AccumuloStorageConfigurationTest.java | 145 +++++++++++++++++-- .../accumulo/pig/AccumuloStorageTest.java | 16 ++ .../java/org/apache/pig/test/MiniCluster.java | 2 +- .../org/apache/pig/test/MiniGenericCluster.java | 2 +- 10 files changed, 221 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 8d0bf0b..d8a339e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .settings /target site/_site +build http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 24449e7..e103468 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ <dependency> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-core</artifactId> - <version>1.5.1-SNAPSHOT</version> + <version>1.5.0</version> </dependency> <dependency> <groupId>joda-time</groupId> @@ -96,7 +96,7 @@ <dependency> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-minicluster</artifactId> - <version>1.5.1-SNAPSHOT</version> + <version>1.5.0</version> <scope>test</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/src/main/java/org/apache/accumulo/pig/AbstractAccumuloStorage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/accumulo/pig/AbstractAccumuloStorage.java b/src/main/java/org/apache/accumulo/pig/AbstractAccumuloStorage.java index b3e91fc..2c21b95 100644 --- a/src/main/java/org/apache/accumulo/pig/AbstractAccumuloStorage.java +++ b/src/main/java/org/apache/accumulo/pig/AbstractAccumuloStorage.java @@ -17,6 +17,8 @@ package org.apache.accumulo.pig; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Collection; @@ -139,7 +141,7 @@ public abstract class AbstractAccumuloStorage extends LoadFunc implements StoreF simpleUnset(conf, entriesToUnset); } else { // If we're running on something else, we have to remove everything and re-add it - replaceUnset(conf, entriesToUnset); + clearUnset(conf, entriesToUnset); } } @@ -149,8 +151,24 @@ public abstract class AbstractAccumuloStorage extends LoadFunc implements StoreF * @param entriesToUnset */ protected void simpleUnset(Configuration conf, Map<String,String> entriesToUnset) { - for (String key : entriesToUnset.keySet()) { - conf.unset(key); + try { + Method unset = conf.getClass().getMethod("unset", String.class); + + for (String key : entriesToUnset.keySet()) { + unset.invoke(conf, key); + } + } catch (NoSuchMethodException e) { + LOG.error("Could not invoke Configuration.unset method", e); + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + LOG.error("Could not invoke Configuration.unset method", e); + throw new RuntimeException(e); + } catch (IllegalArgumentException e) { + LOG.error("Could not invoke Configuration.unset method", e); + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + LOG.error("Could not invoke Configuration.unset method", e); + throw new RuntimeException(e); } } @@ -160,7 +178,7 @@ public abstract class AbstractAccumuloStorage extends LoadFunc implements StoreF * @param conf * @param entriesToUnset */ - protected void replaceUnset(Configuration conf, Map<String,String> entriesToUnset) { + protected void clearUnset(Configuration conf, Map<String,String> entriesToUnset) { // Gets a copy of the entries Iterator<Entry<String,String>> originalEntries = conf.iterator(); conf.clear(); http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/src/main/java/org/apache/accumulo/pig/AccumuloStorage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/accumulo/pig/AccumuloStorage.java b/src/main/java/org/apache/accumulo/pig/AccumuloStorage.java index 742480c..d2be9b6 100644 --- a/src/main/java/org/apache/accumulo/pig/AccumuloStorage.java +++ b/src/main/java/org/apache/accumulo/pig/AccumuloStorage.java @@ -1,3 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package org.apache.accumulo.pig; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/src/main/java/org/apache/accumulo/pig/FORMAT.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/accumulo/pig/FORMAT.java b/src/main/java/org/apache/accumulo/pig/FORMAT.java index a72987a..16a85ca 100644 --- a/src/main/java/org/apache/accumulo/pig/FORMAT.java +++ b/src/main/java/org/apache/accumulo/pig/FORMAT.java @@ -1,3 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package org.apache.accumulo.pig; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/src/test/java/org/apache/accumulo/pig/AccumuloPigClusterTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/accumulo/pig/AccumuloPigClusterTest.java b/src/test/java/org/apache/accumulo/pig/AccumuloPigClusterTest.java index 10b0a2a..d53620d 100644 --- a/src/test/java/org/apache/accumulo/pig/AccumuloPigClusterTest.java +++ b/src/test/java/org/apache/accumulo/pig/AccumuloPigClusterTest.java @@ -1,3 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package org.apache.accumulo.pig; import java.io.File; http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/src/test/java/org/apache/accumulo/pig/AccumuloStorageConfigurationTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/accumulo/pig/AccumuloStorageConfigurationTest.java b/src/test/java/org/apache/accumulo/pig/AccumuloStorageConfigurationTest.java index 079cc0c..c58d26d 100644 --- a/src/test/java/org/apache/accumulo/pig/AccumuloStorageConfigurationTest.java +++ b/src/test/java/org/apache/accumulo/pig/AccumuloStorageConfigurationTest.java @@ -1,3 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package org.apache.accumulo.pig; import java.util.HashMap; @@ -8,6 +24,7 @@ import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class AccumuloStorageConfigurationTest { @@ -41,9 +58,78 @@ public class AccumuloStorageConfigurationTest { return contents; } + @Test + public void testClearEquivalenceStrings() { + Configuration clearCopy = new Configuration(original); + + Assert.assertEquals(getContents(original), getContents(clearCopy)); + + Map<String,String> entriesToUnset = new HashMap<String,String>(); + entriesToUnset.put("string1", "foo"); + entriesToUnset.put("string3", "bar"); + + storage.clearUnset(clearCopy, entriesToUnset); + + Configuration originalCopy = new Configuration(); + for (Entry<String,String> entry : original) { + if (!"string1".equals(entry.getKey()) && !"string3".equals(entry.getKey())) { + originalCopy.set(entry.getKey(), entry.getValue()); + } + } + + Assert.assertEquals(getContents(originalCopy), getContents(clearCopy)); + } + + @Test + public void testClearEquivalenceOnTypes() { + Configuration clearCopy = new Configuration(original); + + Assert.assertEquals(getContents(original), getContents(clearCopy)); + + Map<String,String> entriesToUnset = new HashMap<String,String>(); + entriesToUnset.put("long", "foo"); + entriesToUnset.put("boolean", "bar"); + entriesToUnset.put("integer", "foobar"); + + storage.clearUnset(clearCopy, entriesToUnset); + + Configuration originalCopy = new Configuration(); + for (Entry<String,String> entry : original) { + if (!"long".equals(entry.getKey()) && !"boolean".equals(entry.getKey()) && + !"integer".equals(entry.getKey())) { + originalCopy.set(entry.getKey(), entry.getValue()); + } + } + + Assert.assertEquals(getContents(originalCopy), getContents(clearCopy)); + } + + @Ignore // Ignored until dependency gets updated to Hadoop >=1.2.0 and we have Configuration#unset + @Test + public void testUnsetEquivalenceStrings() { + Configuration unsetCopy = new Configuration(original); + + Assert.assertEquals(getContents(original), getContents(unsetCopy)); + + Map<String,String> entriesToUnset = new HashMap<String,String>(); + entriesToUnset.put("string1", "foo"); + entriesToUnset.put("string3", "bar"); + + storage.simpleUnset(unsetCopy, entriesToUnset); + + Configuration originalCopy = new Configuration(); + for (Entry<String,String> entry : original) { + if (!"string1".equals(entry.getKey()) && !"string2".equals(entry.getKey())) { + originalCopy.set(entry.getKey(), entry.getValue()); + } + } + + Assert.assertEquals(getContents(originalCopy), getContents(unsetCopy)); + } + @Ignore // Ignored until dependency gets updated to Hadoop >=1.2.0 and we have Configuration#unset @Test - public void testEquivalence() { + public void testEquivalenceStrings() { Configuration unsetCopy = new Configuration(original), clearCopy = new Configuration(original); Assert.assertEquals(getContents(unsetCopy), getContents(clearCopy)); @@ -53,24 +139,28 @@ public class AccumuloStorageConfigurationTest { entriesToUnset.put("string3", "bar"); storage.simpleUnset(unsetCopy, entriesToUnset); - storage.replaceUnset(clearCopy, entriesToUnset); + storage.clearUnset(clearCopy, entriesToUnset); Assert.assertEquals(getContents(unsetCopy), getContents(clearCopy)); - Configuration originalCopy = new Configuration(original); - originalCopy.unset("string1"); - originalCopy.unset("string3"); + Configuration originalCopy = new Configuration(); + for (Entry<String,String> entry : original) { + if (!"string1".equals(entry.getKey()) && !"string2".equals(entry.getKey())) { + originalCopy.set(entry.getKey(), entry.getValue()); + } + } Assert.assertEquals(getContents(originalCopy), getContents(unsetCopy)); Assert.assertEquals(getContents(originalCopy), getContents(clearCopy)); } - + @Ignore // Ignored until dependency gets updated to Hadoop >=1.2.0 and we have Configuration#unset @Test public void testEquivalenceOnTypes() { Configuration unsetCopy = new Configuration(original), clearCopy = new Configuration(original); - Assert.assertEquals(getContents(unsetCopy), getContents(clearCopy)); + Assert.assertEquals(getContents(original), getContents(unsetCopy)); + Assert.assertEquals(getContents(original), getContents(clearCopy)); Map<String,String> entriesToUnset = new HashMap<String,String>(); entriesToUnset.put("long", "foo"); @@ -78,17 +168,42 @@ public class AccumuloStorageConfigurationTest { entriesToUnset.put("integer", "foobar"); storage.simpleUnset(unsetCopy, entriesToUnset); - storage.replaceUnset(clearCopy, entriesToUnset); + storage.clearUnset(clearCopy, entriesToUnset); - Assert.assertEquals(getContents(unsetCopy), getContents(clearCopy)); - - Configuration originalCopy = new Configuration(original); - originalCopy.unset("long"); - originalCopy.unset("boolean"); - originalCopy.unset("integer"); + Configuration originalCopy = new Configuration(); + for (Entry<String,String> entry : original) { + if (!"long".equals(entry.getKey()) && !"boolean".equals(entry.getKey()) && + !"integer".equals(entry.getKey())) { + originalCopy.set(entry.getKey(), entry.getValue()); + } + } Assert.assertEquals(getContents(originalCopy), getContents(unsetCopy)); Assert.assertEquals(getContents(originalCopy), getContents(clearCopy)); } - + + @Ignore // Ignored until dependency gets updated to Hadoop >=1.2.0 and we have Configuration#unset + @Test + public void testUnsetEquivalenceOnTypes() { + Configuration unsetCopy = new Configuration(original); + + Assert.assertEquals(getContents(original), getContents(unsetCopy)); + + Map<String,String> entriesToUnset = new HashMap<String,String>(); + entriesToUnset.put("long", "foo"); + entriesToUnset.put("boolean", "bar"); + entriesToUnset.put("integer", "foobar"); + + storage.simpleUnset(unsetCopy, entriesToUnset); + + Configuration originalCopy = new Configuration(); + for (Entry<String,String> entry : original) { + if (!"long".equals(entry.getKey()) && !"boolean".equals(entry.getKey()) && + !"integer".equals(entry.getKey())) { + originalCopy.set(entry.getKey(), entry.getValue()); + } + } + + Assert.assertEquals(getContents(originalCopy), getContents(unsetCopy)); + } } http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/src/test/java/org/apache/accumulo/pig/AccumuloStorageTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/accumulo/pig/AccumuloStorageTest.java b/src/test/java/org/apache/accumulo/pig/AccumuloStorageTest.java index aa88191..d102d64 100644 --- a/src/test/java/org/apache/accumulo/pig/AccumuloStorageTest.java +++ b/src/test/java/org/apache/accumulo/pig/AccumuloStorageTest.java @@ -1,3 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package org.apache.accumulo.pig; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/src/test/java/org/apache/pig/test/MiniCluster.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pig/test/MiniCluster.java b/src/test/java/org/apache/pig/test/MiniCluster.java index 64467ae..aacd30f 100644 --- a/src/test/java/org/apache/pig/test/MiniCluster.java +++ b/src/test/java/org/apache/pig/test/MiniCluster.java @@ -1,4 +1,3 @@ -package org.apache.pig.test; /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -16,6 +15,7 @@ package org.apache.pig.test; * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.pig.test; import java.io.File; http://git-wip-us.apache.org/repos/asf/accumulo-pig/blob/6820d50a/src/test/java/org/apache/pig/test/MiniGenericCluster.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pig/test/MiniGenericCluster.java b/src/test/java/org/apache/pig/test/MiniGenericCluster.java index 584631a..b4ea84f 100644 --- a/src/test/java/org/apache/pig/test/MiniGenericCluster.java +++ b/src/test/java/org/apache/pig/test/MiniGenericCluster.java @@ -1,4 +1,3 @@ -package org.apache.pig.test; /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -16,6 +15,7 @@ package org.apache.pig.test; * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.pig.test; import java.io.*;
