http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/test/proxy/TBinaryProxyIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/TBinaryProxyIT.java b/test/src/test/java/org/apache/accumulo/test/proxy/TBinaryProxyIT.java new file mode 100644 index 0000000..6359d1e --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/proxy/TBinaryProxyIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.proxy; + +import org.apache.thrift.protocol.TBinaryProtocol; +import org.junit.BeforeClass; + +/** + * + */ +public class TBinaryProxyIT extends SimpleProxyBase { + + @BeforeClass + public static void setProtocol() throws Exception { + SimpleProxyBase.factory = new TBinaryProtocol.Factory(); + setUpProxy(); + } + +}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/test/proxy/TCompactProxyIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/TCompactProxyIT.java b/test/src/test/java/org/apache/accumulo/test/proxy/TCompactProxyIT.java new file mode 100644 index 0000000..a92414a --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/proxy/TCompactProxyIT.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.proxy; + +import org.apache.thrift.protocol.TCompactProtocol; +import org.junit.BeforeClass; + +/** + * + */ +public class TCompactProxyIT extends SimpleProxyBase { + + @BeforeClass + public static void setProtocol() throws Exception { + SimpleProxyBase.factory = new TCompactProtocol.Factory(); + setUpProxy(); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/test/proxy/TJsonProtocolProxyIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/TJsonProtocolProxyIT.java b/test/src/test/java/org/apache/accumulo/test/proxy/TJsonProtocolProxyIT.java new file mode 100644 index 0000000..5fcbf53 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/proxy/TJsonProtocolProxyIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.proxy; + +import org.apache.thrift.protocol.TJSONProtocol; +import org.junit.BeforeClass; + +/** + * + */ +public class TJsonProtocolProxyIT extends SimpleProxyBase { + + @BeforeClass + public static void setProtocol() throws Exception { + SimpleProxyBase.factory = new TJSONProtocol.Factory(); + setUpProxy(); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/test/proxy/TTupleProxyIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/TTupleProxyIT.java b/test/src/test/java/org/apache/accumulo/test/proxy/TTupleProxyIT.java new file mode 100644 index 0000000..cdecf2c --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/proxy/TTupleProxyIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.proxy; + +import org.apache.thrift.protocol.TTupleProtocol; +import org.junit.BeforeClass; + +/** + * + */ +public class TTupleProxyIT extends SimpleProxyBase { + + @BeforeClass + public static void setProtocol() throws Exception { + SimpleProxyBase.factory = new TTupleProtocol.Factory(); + setUpProxy(); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java b/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java new file mode 100644 index 0000000..caed876 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.proxy; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Properties; + +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.proxy.Proxy; +import org.apache.accumulo.proxy.TestProxyClient; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.server.TServer; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.net.HostAndPort; + +public class TestProxyInstanceOperations { + private static final Logger log = LoggerFactory.getLogger(TestProxyInstanceOperations.class); + + protected static TServer proxy; + protected static TestProxyClient tpc; + protected static ByteBuffer userpass; + protected static final int port = 10197; + + @BeforeClass + public static void setup() throws Exception { + Properties prop = new Properties(); + prop.setProperty("useMockInstance", "true"); + prop.put("tokenClass", PasswordToken.class.getName()); + + proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), new TCompactProtocol.Factory(), prop).server; + log.info("Waiting for proxy to start"); + while (!proxy.isServing()) { + Thread.sleep(500); + } + log.info("Proxy started"); + tpc = new TestProxyClient("localhost", port); + userpass = tpc.proxy().login("root", Collections.singletonMap("password", "")); + } + + @AfterClass + public static void tearDown() throws InterruptedException { + proxy.stop(); + } + + @Test + public void properties() throws TException { + tpc.proxy().setProperty(userpass, "test.systemprop", "whistletips"); + + assertEquals(tpc.proxy().getSystemConfiguration(userpass).get("test.systemprop"), "whistletips"); + tpc.proxy().removeProperty(userpass, "test.systemprop"); + assertNull(tpc.proxy().getSystemConfiguration(userpass).get("test.systemprop")); + + } + + @Test + public void testClassLoad() throws TException { + assertTrue(tpc.proxy().testClassLoad(userpass, "org.apache.accumulo.core.iterators.user.RegExFilter", "org.apache.accumulo.core.iterators.Filter")); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java b/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java new file mode 100644 index 0000000..5eb9500 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java @@ -0,0 +1,469 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.proxy; + +import static org.junit.Assert.assertEquals; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.iterators.user.RegExFilter; +import org.apache.accumulo.proxy.Proxy; +import org.apache.accumulo.proxy.TestProxyClient; +import org.apache.accumulo.proxy.Util; +import org.apache.accumulo.proxy.thrift.BatchScanOptions; +import org.apache.accumulo.proxy.thrift.ColumnUpdate; +import org.apache.accumulo.proxy.thrift.IteratorSetting; +import org.apache.accumulo.proxy.thrift.Key; +import org.apache.accumulo.proxy.thrift.KeyValue; +import org.apache.accumulo.proxy.thrift.Range; +import org.apache.accumulo.proxy.thrift.ScanColumn; +import org.apache.accumulo.proxy.thrift.ScanOptions; +import org.apache.accumulo.proxy.thrift.ScanResult; +import org.apache.accumulo.proxy.thrift.TimeType; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.server.TServer; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.net.HostAndPort; + +public class TestProxyReadWrite { + protected static TServer proxy; + protected static TestProxyClient tpc; + protected static ByteBuffer userpass; + protected static final int port = 10194; + protected static final String testtable = "testtable"; + + @BeforeClass + public static void setup() throws Exception { + Properties prop = new Properties(); + prop.setProperty("useMockInstance", "true"); + prop.put("tokenClass", PasswordToken.class.getName()); + + proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), new TCompactProtocol.Factory(), prop).server; + tpc = new TestProxyClient("localhost", port); + userpass = tpc.proxy().login("root", Collections.singletonMap("password", "")); + } + + @AfterClass + public static void tearDown() throws InterruptedException { + proxy.stop(); + } + + @Before + public void makeTestTable() throws Exception { + tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS); + } + + @After + public void deleteTestTable() throws Exception { + tpc.proxy().deleteTable(userpass, testtable); + } + + private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String value) { + ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes())); + update.setValue(value.getBytes()); + mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update)); + } + + private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String vis, String value) { + ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes())); + update.setValue(value.getBytes()); + update.setColVisibility(vis.getBytes()); + mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update)); + } + + /** + * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a range so only the entries between -Inf...5 come back (there should be + * 50,000) + */ + @Test + public void readWriteBatchOneShotWithRange() throws Exception { + int maxInserts = 100000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$05d"; + for (int i = 0; i < maxInserts; i++) { + addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + tpc.proxy().updateAndFlush(userpass, testtable, mutations); + mutations.clear(); + } + } + + Key stop = new Key(); + stop.setRow("5".getBytes()); + BatchScanOptions options = new BatchScanOptions(); + options.ranges = Collections.singletonList(new Range(null, false, stop, false)); + String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + i += kvList.getResultsSize(); + hasNext = kvList.isMore(); + } + assertEquals(i, 50000); + } + + /** + * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily so only the entries with specified column family come back + * (there should be 50,000) + */ + @Test + public void readWriteBatchOneShotWithColumnFamilyOnly() throws Exception { + int maxInserts = 100000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$05d"; + for (int i = 0; i < maxInserts; i++) { + + addMutation(mutations, String.format(format, i), "cf" + (i % 2), "cq" + (i % 2), Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + tpc.proxy().updateAndFlush(userpass, testtable, mutations); + mutations.clear(); + } + } + + BatchScanOptions options = new BatchScanOptions(); + + ScanColumn sc = new ScanColumn(); + sc.colFamily = ByteBuffer.wrap("cf0".getBytes()); + + options.columns = Collections.singletonList(sc); + String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + i += kvList.getResultsSize(); + hasNext = kvList.isMore(); + } + assertEquals(i, 50000); + } + + /** + * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily + columnQualififer so only the entries with specified column + * come back (there should be 50,000) + */ + @Test + public void readWriteBatchOneShotWithFullColumn() throws Exception { + int maxInserts = 100000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$05d"; + for (int i = 0; i < maxInserts; i++) { + + addMutation(mutations, String.format(format, i), "cf" + (i % 2), "cq" + (i % 2), Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + tpc.proxy().updateAndFlush(userpass, testtable, mutations); + mutations.clear(); + } + } + + BatchScanOptions options = new BatchScanOptions(); + + ScanColumn sc = new ScanColumn(); + sc.colFamily = ByteBuffer.wrap("cf0".getBytes()); + sc.colQualifier = ByteBuffer.wrap("cq0".getBytes()); + + options.columns = Collections.singletonList(sc); + String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + i += kvList.getResultsSize(); + hasNext = kvList.isMore(); + } + assertEquals(i, 50000); + } + + /** + * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so only the even numbers come back. + */ + @Test + public void readWriteBatchOneShotWithFilterIterator() throws Exception { + int maxInserts = 10000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$05d"; + for (int i = 0; i < maxInserts; i++) { + addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + tpc.proxy().updateAndFlush(userpass, testtable, mutations); + mutations.clear(); + } + + } + + String regex = ".*[02468]"; + + org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class); + RegExFilter.setRegexs(is, regex, null, null, null, false); + + IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is); + ScanOptions opts = new ScanOptions(); + opts.iterators = Collections.singletonList(pis); + String cookie = tpc.proxy().createScanner(userpass, testtable, opts); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + for (KeyValue kv : kvList.getResults()) { + assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i); + + i += 2; + } + hasNext = kvList.isMore(); + } + } + + @Test + public void readWriteOneShotWithRange() throws Exception { + int maxInserts = 100000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$05d"; + for (int i = 0; i < maxInserts; i++) { + addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + tpc.proxy().updateAndFlush(userpass, testtable, mutations); + mutations.clear(); + } + } + + Key stop = new Key(); + stop.setRow("5".getBytes()); + ScanOptions opts = new ScanOptions(); + opts.range = new Range(null, false, stop, false); + String cookie = tpc.proxy().createScanner(userpass, testtable, opts); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + i += kvList.getResultsSize(); + hasNext = kvList.isMore(); + } + assertEquals(i, 50000); + } + + /** + * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so only the even numbers come back. + */ + @Test + public void readWriteOneShotWithFilterIterator() throws Exception { + int maxInserts = 10000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$05d"; + for (int i = 0; i < maxInserts; i++) { + addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + + tpc.proxy().updateAndFlush(userpass, testtable, mutations); + mutations.clear(); + + } + + } + + String regex = ".*[02468]"; + + org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class); + RegExFilter.setRegexs(is, regex, null, null, null, false); + + IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is); + ScanOptions opts = new ScanOptions(); + opts.iterators = Collections.singletonList(pis); + String cookie = tpc.proxy().createScanner(userpass, testtable, opts); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + for (KeyValue kv : kvList.getResults()) { + assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i); + + i += 2; + } + hasNext = kvList.isMore(); + } + } + + // @Test + // This test takes kind of a long time. Enable it if you think you may have memory issues. + public void manyWritesAndReads() throws Exception { + int maxInserts = 1000000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$06d"; + String writer = tpc.proxy().createWriter(userpass, testtable, null); + for (int i = 0; i < maxInserts; i++) { + addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + + tpc.proxy().update(writer, mutations); + mutations.clear(); + + } + + } + + tpc.proxy().flush(writer); + tpc.proxy().closeWriter(writer); + + String cookie = tpc.proxy().createScanner(userpass, testtable, null); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + for (KeyValue kv : kvList.getResults()) { + assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i); + i++; + } + hasNext = kvList.isMore(); + if (hasNext) + assertEquals(k, kvList.getResults().size()); + } + assertEquals(maxInserts, i); + } + + @Test + public void asynchReadWrite() throws Exception { + int maxInserts = 10000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$05d"; + String writer = tpc.proxy().createWriter(userpass, testtable, null); + for (int i = 0; i < maxInserts; i++) { + addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + tpc.proxy().update(writer, mutations); + mutations.clear(); + } + } + + tpc.proxy().flush(writer); + tpc.proxy().closeWriter(writer); + + String regex = ".*[02468]"; + + org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class); + RegExFilter.setRegexs(is, regex, null, null, null, false); + + IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is); + ScanOptions opts = new ScanOptions(); + opts.iterators = Collections.singletonList(pis); + String cookie = tpc.proxy().createScanner(userpass, testtable, opts); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + int numRead = 0; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + for (KeyValue kv : kvList.getResults()) { + assertEquals(i, Integer.parseInt(new String(kv.getKey().getRow()))); + numRead++; + i += 2; + } + hasNext = kvList.isMore(); + } + assertEquals(maxInserts / 2, numRead); + } + + @Test + public void testVisibility() throws Exception { + + Set<ByteBuffer> auths = new HashSet<ByteBuffer>(); + auths.add(ByteBuffer.wrap("even".getBytes())); + tpc.proxy().changeUserAuthorizations(userpass, "root", auths); + + int maxInserts = 10000; + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + String format = "%1$05d"; + String writer = tpc.proxy().createWriter(userpass, testtable, null); + for (int i = 0; i < maxInserts; i++) { + if (i % 2 == 0) + addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "even", Util.randString(10)); + else + addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "odd", Util.randString(10)); + + if (i % 1000 == 0 || i == maxInserts - 1) { + tpc.proxy().update(writer, mutations); + mutations.clear(); + } + } + + tpc.proxy().flush(writer); + tpc.proxy().closeWriter(writer); + ScanOptions opts = new ScanOptions(); + opts.authorizations = auths; + String cookie = tpc.proxy().createScanner(userpass, testtable, opts); + + int i = 0; + boolean hasNext = true; + + int k = 1000; + int numRead = 0; + while (hasNext) { + ScanResult kvList = tpc.proxy().nextK(cookie, k); + for (KeyValue kv : kvList.getResults()) { + assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i); + i += 2; + numRead++; + } + hasNext = kvList.isMore(); + + } + assertEquals(maxInserts / 2, numRead); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java b/test/src/test/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java new file mode 100644 index 0000000..ec3ddf3 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.proxy; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; + +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.util.ByteBufferUtil; +import org.apache.accumulo.proxy.Proxy; +import org.apache.accumulo.proxy.TestProxyClient; +import org.apache.accumulo.proxy.thrift.SystemPermission; +import org.apache.accumulo.proxy.thrift.TablePermission; +import org.apache.accumulo.proxy.thrift.TimeType; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.server.TServer; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.net.HostAndPort; + +public class TestProxySecurityOperations { + protected static TServer proxy; + protected static TestProxyClient tpc; + protected static ByteBuffer userpass; + protected static final int port = 10196; + protected static final String testtable = "testtable"; + protected static final String testuser = "VonJines"; + protected static final ByteBuffer testpw = ByteBuffer.wrap("fiveones".getBytes()); + + @BeforeClass + public static void setup() throws Exception { + Properties prop = new Properties(); + prop.setProperty("useMockInstance", "true"); + prop.put("tokenClass", PasswordToken.class.getName()); + + proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), new TCompactProtocol.Factory(), prop).server; + while (!proxy.isServing()) { + Thread.sleep(500); + } + tpc = new TestProxyClient("localhost", port); + userpass = tpc.proxy().login("root", Collections.singletonMap("password", "")); + } + + @AfterClass + public static void tearDown() throws InterruptedException { + proxy.stop(); + } + + @Before + public void makeTestTableAndUser() throws Exception { + tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS); + tpc.proxy().createLocalUser(userpass, testuser, testpw); + } + + @After + public void deleteTestTable() throws Exception { + tpc.proxy().deleteTable(userpass, testtable); + tpc.proxy().dropLocalUser(userpass, testuser); + } + + @Test + public void create() throws TException { + tpc.proxy().createLocalUser(userpass, testuser + "2", testpw); + assertTrue(tpc.proxy().listLocalUsers(userpass).contains(testuser + "2")); + tpc.proxy().dropLocalUser(userpass, testuser + "2"); + assertTrue(!tpc.proxy().listLocalUsers(userpass).contains(testuser + "2")); + } + + @Test + public void authenticate() throws TException { + assertTrue(tpc.proxy().authenticateUser(userpass, testuser, bb2pp(testpw))); + assertFalse(tpc.proxy().authenticateUser(userpass, "EvilUser", bb2pp(testpw))); + + tpc.proxy().changeLocalUserPassword(userpass, testuser, ByteBuffer.wrap("newpass".getBytes())); + assertFalse(tpc.proxy().authenticateUser(userpass, testuser, bb2pp(testpw))); + assertTrue(tpc.proxy().authenticateUser(userpass, testuser, bb2pp(ByteBuffer.wrap("newpass".getBytes())))); + + } + + @Test + public void tablePermissions() throws TException { + tpc.proxy().grantTablePermission(userpass, testuser, testtable, TablePermission.ALTER_TABLE); + assertTrue(tpc.proxy().hasTablePermission(userpass, testuser, testtable, TablePermission.ALTER_TABLE)); + + tpc.proxy().revokeTablePermission(userpass, testuser, testtable, TablePermission.ALTER_TABLE); + assertFalse(tpc.proxy().hasTablePermission(userpass, testuser, testtable, TablePermission.ALTER_TABLE)); + + } + + @Test + public void systemPermissions() throws TException { + tpc.proxy().grantSystemPermission(userpass, testuser, SystemPermission.ALTER_USER); + assertTrue(tpc.proxy().hasSystemPermission(userpass, testuser, SystemPermission.ALTER_USER)); + + tpc.proxy().revokeSystemPermission(userpass, testuser, SystemPermission.ALTER_USER); + assertFalse(tpc.proxy().hasSystemPermission(userpass, testuser, SystemPermission.ALTER_USER)); + + } + + @Test + public void auths() throws TException { + HashSet<ByteBuffer> newauths = new HashSet<ByteBuffer>(); + newauths.add(ByteBuffer.wrap("BBR".getBytes())); + newauths.add(ByteBuffer.wrap("Barney".getBytes())); + tpc.proxy().changeUserAuthorizations(userpass, testuser, newauths); + List<ByteBuffer> actualauths = tpc.proxy().getUserAuthorizations(userpass, testuser); + assertEquals(actualauths.size(), newauths.size()); + + for (ByteBuffer auth : actualauths) { + assertTrue(newauths.contains(auth)); + } + } + + private Map<String,String> bb2pp(ByteBuffer cf) { + Map<String,String> toRet = new TreeMap<String,String>(); + toRet.put("password", ByteBufferUtil.toString(cf)); + return toRet; + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java b/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java new file mode 100644 index 0000000..7da2f63 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.proxy; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.proxy.Proxy; +import org.apache.accumulo.proxy.TestProxyClient; +import org.apache.accumulo.proxy.thrift.ColumnUpdate; +import org.apache.accumulo.proxy.thrift.TimeType; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.server.TServer; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.net.HostAndPort; + +public class TestProxyTableOperations { + + protected static TServer proxy; + protected static TestProxyClient tpc; + protected static ByteBuffer userpass; + protected static final int port = 10195; + protected static final String testtable = "testtable"; + + @BeforeClass + public static void setup() throws Exception { + Properties prop = new Properties(); + prop.setProperty("useMockInstance", "true"); + prop.put("tokenClass", PasswordToken.class.getName()); + + proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), new TCompactProtocol.Factory(), prop).server; + while (!proxy.isServing()) { + Thread.sleep(500); + } + tpc = new TestProxyClient("localhost", port); + userpass = tpc.proxy().login("root", Collections.singletonMap("password", "")); + } + + @AfterClass + public static void tearDown() throws InterruptedException { + proxy.stop(); + } + + @Before + public void makeTestTable() throws Exception { + tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS); + } + + @After + public void deleteTestTable() throws Exception { + tpc.proxy().deleteTable(userpass, testtable); + } + + @Test + public void createExistsDelete() throws TException { + assertFalse(tpc.proxy().tableExists(userpass, "testtable2")); + tpc.proxy().createTable(userpass, "testtable2", true, TimeType.MILLIS); + assertTrue(tpc.proxy().tableExists(userpass, "testtable2")); + tpc.proxy().deleteTable(userpass, "testtable2"); + assertFalse(tpc.proxy().tableExists(userpass, "testtable2")); + } + + @Test + public void listRename() throws TException { + assertFalse(tpc.proxy().tableExists(userpass, "testtable2")); + tpc.proxy().renameTable(userpass, testtable, "testtable2"); + assertTrue(tpc.proxy().tableExists(userpass, "testtable2")); + tpc.proxy().renameTable(userpass, "testtable2", testtable); + assertTrue(tpc.proxy().listTables(userpass).contains("testtable")); + + } + + // This test does not yet function because the backing Mock instance does not yet support merging + @Test + public void merge() throws TException { + Set<ByteBuffer> splits = new HashSet<ByteBuffer>(); + splits.add(ByteBuffer.wrap("a".getBytes())); + splits.add(ByteBuffer.wrap("c".getBytes())); + splits.add(ByteBuffer.wrap("z".getBytes())); + tpc.proxy().addSplits(userpass, testtable, splits); + + tpc.proxy().mergeTablets(userpass, testtable, ByteBuffer.wrap("b".getBytes()), ByteBuffer.wrap("d".getBytes())); + + splits.remove(ByteBuffer.wrap("c".getBytes())); + + List<ByteBuffer> tableSplits = tpc.proxy().listSplits(userpass, testtable, 10); + + for (ByteBuffer split : tableSplits) + assertTrue(splits.contains(split)); + assertTrue(tableSplits.size() == splits.size()); + + } + + @Test + public void splits() throws TException { + Set<ByteBuffer> splits = new HashSet<ByteBuffer>(); + splits.add(ByteBuffer.wrap("a".getBytes())); + splits.add(ByteBuffer.wrap("b".getBytes())); + splits.add(ByteBuffer.wrap("z".getBytes())); + tpc.proxy().addSplits(userpass, testtable, splits); + + List<ByteBuffer> tableSplits = tpc.proxy().listSplits(userpass, testtable, 10); + + for (ByteBuffer split : tableSplits) + assertTrue(splits.contains(split)); + assertTrue(tableSplits.size() == splits.size()); + } + + @Test + public void constraints() throws TException { + int cid = tpc.proxy().addConstraint(userpass, testtable, "org.apache.accumulo.TestConstraint"); + Map<String,Integer> constraints = tpc.proxy().listConstraints(userpass, testtable); + assertEquals((int) constraints.get("org.apache.accumulo.TestConstraint"), cid); + tpc.proxy().removeConstraint(userpass, testtable, cid); + constraints = tpc.proxy().listConstraints(userpass, testtable); + assertNull(constraints.get("org.apache.accumulo.TestConstraint")); + } + + @Test + public void localityGroups() throws TException { + Map<String,Set<String>> groups = new HashMap<String,Set<String>>(); + Set<String> group1 = new HashSet<String>(); + group1.add("cf1"); + groups.put("group1", group1); + Set<String> group2 = new HashSet<String>(); + group2.add("cf2"); + group2.add("cf3"); + groups.put("group2", group2); + tpc.proxy().setLocalityGroups(userpass, testtable, groups); + + Map<String,Set<String>> actualGroups = tpc.proxy().getLocalityGroups(userpass, testtable); + + assertEquals(groups.size(), actualGroups.size()); + for (String groupName : groups.keySet()) { + assertTrue(actualGroups.containsKey(groupName)); + assertEquals(groups.get(groupName).size(), actualGroups.get(groupName).size()); + for (String cf : groups.get(groupName)) { + assertTrue(actualGroups.get(groupName).contains(cf)); + } + } + } + + @Test + public void tableProperties() throws TException { + tpc.proxy().setTableProperty(userpass, testtable, "test.property1", "wharrrgarbl"); + assertEquals(tpc.proxy().getTableProperties(userpass, testtable).get("test.property1"), "wharrrgarbl"); + tpc.proxy().removeTableProperty(userpass, testtable, "test.property1"); + assertNull(tpc.proxy().getTableProperties(userpass, testtable).get("test.property1")); + } + + private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String value) { + ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes())); + update.setValue(value.getBytes()); + mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update)); + } + + @Test + public void tableOperationsRowMethods() throws TException { + Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>(); + for (int i = 0; i < 10; i++) { + addMutation(mutations, "" + i, "cf", "cq", ""); + } + tpc.proxy().updateAndFlush(userpass, testtable, mutations); + + assertEquals(tpc.proxy().getMaxRow(userpass, testtable, null, null, true, null, true), ByteBuffer.wrap("9".getBytes())); + + tpc.proxy().deleteRows(userpass, testtable, ByteBuffer.wrap("51".getBytes()), ByteBuffer.wrap("99".getBytes())); + assertEquals(tpc.proxy().getMaxRow(userpass, testtable, null, null, true, null, true), ByteBuffer.wrap("5".getBytes())); + } + +}
