http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/base/src/test/java/org/apache/accumulo/server/rpc/TCredentialsUpdatingInvocationHandlerTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/rpc/TCredentialsUpdatingInvocationHandlerTest.java b/server/base/src/test/java/org/apache/accumulo/server/rpc/TCredentialsUpdatingInvocationHandlerTest.java index 1b87a9e..4f7cbba 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/rpc/TCredentialsUpdatingInvocationHandlerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/rpc/TCredentialsUpdatingInvocationHandlerTest.java @@ -116,6 +116,18 @@ public class TCredentialsUpdatingInvocationHandlerTest { proxy.updateArgs(new Object[] {new Object(), tcreds}); } + @SuppressWarnings("deprecation") + @Test + public void testAllowedAnyImpersonationForAnyUser() throws Exception { + final String proxyServer = "proxy"; + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + proxyServer + ".users", "*"); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + proxyServer + ".hosts", "*"); + proxy = new TCredentialsUpdatingInvocationHandler<>(new Object(), conf); + TCredentials tcreds = new TCredentials("client", KerberosToken.class.getName(), ByteBuffer.allocate(0), UUID.randomUUID().toString()); + UGIAssumingProcessor.rpcPrincipal.set(proxyServer); + proxy.updateArgs(new Object[] {new Object(), tcreds}); + } + @Test public void testAllowedAnyImpersonationForAnyUserNewConfig() throws Exception { final String proxyServer = "proxy"; @@ -127,6 +139,20 @@ public class TCredentialsUpdatingInvocationHandlerTest { proxy.updateArgs(new Object[] {new Object(), tcreds}); } + @SuppressWarnings("deprecation") + @Test + public void testAllowedImpersonationForSpecificUsers() throws Exception { + final String proxyServer = "proxy"; + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + proxyServer + ".users", "client1,client2"); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + proxyServer + ".hosts", "*"); + proxy = new TCredentialsUpdatingInvocationHandler<>(new Object(), conf); + TCredentials tcreds = new TCredentials("client1", KerberosToken.class.getName(), ByteBuffer.allocate(0), UUID.randomUUID().toString()); + UGIAssumingProcessor.rpcPrincipal.set(proxyServer); + proxy.updateArgs(new Object[] {new Object(), tcreds}); + tcreds = new TCredentials("client2", KerberosToken.class.getName(), ByteBuffer.allocate(0), UUID.randomUUID().toString()); + proxy.updateArgs(new Object[] {new Object(), tcreds}); + } + @Test public void testAllowedImpersonationForSpecificUsersNewConfig() throws Exception { final String proxyServer = "proxy"; @@ -140,6 +166,19 @@ public class TCredentialsUpdatingInvocationHandlerTest { proxy.updateArgs(new Object[] {new Object(), tcreds}); } + @SuppressWarnings("deprecation") + @Test(expected = ThriftSecurityException.class) + public void testDisallowedImpersonationForUser() throws Exception { + final String proxyServer = "proxy"; + // let "otherproxy" impersonate, but not "proxy" + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + "otherproxy" + ".users", "*"); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + "otherproxy" + ".hosts", "*"); + proxy = new TCredentialsUpdatingInvocationHandler<>(new Object(), conf); + TCredentials tcreds = new TCredentials("client", KerberosToken.class.getName(), ByteBuffer.allocate(0), UUID.randomUUID().toString()); + UGIAssumingProcessor.rpcPrincipal.set(proxyServer); + proxy.updateArgs(new Object[] {new Object(), tcreds}); + } + @Test(expected = ThriftSecurityException.class) public void testDisallowedImpersonationForUserNewConfig() throws Exception { final String proxyServer = "proxy"; @@ -152,6 +191,21 @@ public class TCredentialsUpdatingInvocationHandlerTest { proxy.updateArgs(new Object[] {new Object(), tcreds}); } + @SuppressWarnings("deprecation") + @Test(expected = ThriftSecurityException.class) + public void testDisallowedImpersonationForMultipleUsers() throws Exception { + final String proxyServer = "proxy"; + // let "otherproxy" impersonate, but not "proxy" + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + "otherproxy1" + ".users", "*"); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + "otherproxy1" + ".hosts", "*"); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + "otherproxy2" + ".users", "client1,client2"); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + "otherproxy2" + ".hosts", "*"); + proxy = new TCredentialsUpdatingInvocationHandler<>(new Object(), conf); + TCredentials tcreds = new TCredentials("client1", KerberosToken.class.getName(), ByteBuffer.allocate(0), UUID.randomUUID().toString()); + UGIAssumingProcessor.rpcPrincipal.set(proxyServer); + proxy.updateArgs(new Object[] {new Object(), tcreds}); + } + @Test(expected = ThriftSecurityException.class) public void testDisallowedImpersonationForMultipleUsersNewConfig() throws Exception { final String proxyServer = "proxy"; @@ -164,6 +218,19 @@ public class TCredentialsUpdatingInvocationHandlerTest { proxy.updateArgs(new Object[] {new Object(), tcreds}); } + @SuppressWarnings("deprecation") + @Test + public void testAllowedImpersonationFromSpecificHost() throws Exception { + final String proxyServer = "proxy", client = "client", host = "host.domain.com"; + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + proxyServer + ".users", client); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + proxyServer + ".hosts", host); + proxy = new TCredentialsUpdatingInvocationHandler<>(new Object(), conf); + TCredentials tcreds = new TCredentials("client", KerberosToken.class.getName(), ByteBuffer.allocate(0), UUID.randomUUID().toString()); + UGIAssumingProcessor.rpcPrincipal.set(proxyServer); + TServerUtils.clientAddress.set(host); + proxy.updateArgs(new Object[] {new Object(), tcreds}); + } + @Test public void testAllowedImpersonationFromSpecificHostNewConfig() throws Exception { final String proxyServer = "proxy", client = "client", host = "host.domain.com"; @@ -176,6 +243,20 @@ public class TCredentialsUpdatingInvocationHandlerTest { proxy.updateArgs(new Object[] {new Object(), tcreds}); } + @SuppressWarnings("deprecation") + @Test(expected = ThriftSecurityException.class) + public void testDisallowedImpersonationFromSpecificHost() throws Exception { + final String proxyServer = "proxy", client = "client", host = "host.domain.com"; + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + proxyServer + ".users", client); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + proxyServer + ".hosts", host); + proxy = new TCredentialsUpdatingInvocationHandler<>(new Object(), conf); + TCredentials tcreds = new TCredentials("client", KerberosToken.class.getName(), ByteBuffer.allocate(0), UUID.randomUUID().toString()); + UGIAssumingProcessor.rpcPrincipal.set(proxyServer); + // The RPC came from a different host than is allowed + TServerUtils.clientAddress.set("otherhost.domain.com"); + proxy.updateArgs(new Object[] {new Object(), tcreds}); + } + @Test(expected = ThriftSecurityException.class) public void testDisallowedImpersonationFromSpecificHostNewConfig() throws Exception { final String proxyServer = "proxy", client = "client", host = "host.domain.com";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/base/src/test/java/org/apache/accumulo/server/security/UserImpersonationTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/security/UserImpersonationTest.java b/server/base/src/test/java/org/apache/accumulo/server/security/UserImpersonationTest.java index 714a1c1..5b4e1b0 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/security/UserImpersonationTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/security/UserImpersonationTest.java @@ -67,6 +67,19 @@ public class UserImpersonationTest { }; } + void setValidHosts(String user, String hosts) { + setUsersOrHosts(user, ".hosts", hosts); + } + + void setValidUsers(String user, String users) { + setUsersOrHosts(user, ".users", users); + } + + @SuppressWarnings("deprecation") + void setUsersOrHosts(String user, String suffix, String value) { + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + user + suffix, value); + } + void setValidHostsNewConfig(String user, String... hosts) { cc.set(Property.INSTANCE_RPC_SASL_ALLOWED_HOST_IMPERSONATION.getKey(), Joiner.on(';').join(hosts)); } @@ -83,6 +96,23 @@ public class UserImpersonationTest { } @Test + public void testAnyUserAndHosts() { + String server = "server"; + setValidHosts(server, "*"); + setValidUsers(server, "*"); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server); + assertNotNull(uwh); + + assertTrue(uwh.acceptsAllHosts()); + assertTrue(uwh.acceptsAllUsers()); + + assertEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + } + + @Test public void testAnyUserAndHostsNewConfig() { String server = "server"; setValidHostsNewConfig(server, "*"); @@ -100,6 +130,22 @@ public class UserImpersonationTest { } @Test + public void testNoHostByDefault() { + String server = "server"; + setValidUsers(server, "*"); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server); + assertNotNull(uwh); + + assertFalse(uwh.acceptsAllHosts()); + assertTrue(uwh.acceptsAllUsers()); + + assertNotEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + } + + @Test public void testNoHostByDefaultNewConfig() { String server = "server"; setValidUsersNewConfig(ImmutableMap.of(server, "*")); @@ -116,6 +162,22 @@ public class UserImpersonationTest { } @Test + public void testNoUsersByDefault() { + String server = "server"; + setValidHosts(server, "*"); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server); + assertNotNull(uwh); + + assertTrue(uwh.acceptsAllHosts()); + assertFalse(uwh.acceptsAllUsers()); + + assertEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertNotEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + } + + @Test public void testNoUsersByDefaultNewConfig() { String server = "server"; setValidHostsNewConfig(server, "*"); @@ -126,6 +188,29 @@ public class UserImpersonationTest { } @Test + public void testSingleUserAndHost() { + String server = "server", host = "single_host.domain.com", client = "single_client"; + setValidHosts(server, host); + setValidUsers(server, client); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server); + assertNotNull(uwh); + + assertFalse(uwh.acceptsAllHosts()); + assertFalse(uwh.acceptsAllUsers()); + + assertNotEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertNotEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + + assertTrue(uwh.getUsers().contains(client)); + assertTrue(uwh.getHosts().contains(host)); + + assertFalse(uwh.getUsers().contains("some_other_user")); + assertFalse(uwh.getHosts().contains("other_host.domain.com")); + } + + @Test public void testSingleUserAndHostNewConfig() { String server = "server", host = "single_host.domain.com", client = "single_client"; setValidHostsNewConfig(server, host); @@ -149,6 +234,28 @@ public class UserImpersonationTest { } @Test + public void testMultipleExplicitUsers() { + String server = "server", client1 = "client1", client2 = "client2", client3 = "client3"; + setValidHosts(server, "*"); + setValidUsers(server, Joiner.on(',').join(client1, client2, client3)); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server); + assertNotNull(uwh); + + assertTrue(uwh.acceptsAllHosts()); + assertFalse(uwh.acceptsAllUsers()); + + assertEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertNotEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + + assertTrue(uwh.getUsers().contains(client1)); + assertTrue(uwh.getUsers().contains(client2)); + assertTrue(uwh.getUsers().contains(client3)); + assertFalse(uwh.getUsers().contains("other_client")); + } + + @Test public void testMultipleExplicitUsersNewConfig() { String server = "server", client1 = "client1", client2 = "client2", client3 = "client3"; setValidHostsNewConfig(server, "*"); @@ -171,6 +278,28 @@ public class UserImpersonationTest { } @Test + public void testMultipleExplicitHosts() { + String server = "server", host1 = "host1", host2 = "host2", host3 = "host3"; + setValidHosts(server, Joiner.on(',').join(host1, host2, host3)); + setValidUsers(server, "*"); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server); + assertNotNull(uwh); + + assertFalse(uwh.acceptsAllHosts()); + assertTrue(uwh.acceptsAllUsers()); + + assertNotEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + + assertTrue(uwh.getHosts().contains(host1)); + assertTrue(uwh.getHosts().contains(host2)); + assertTrue(uwh.getHosts().contains(host3)); + assertFalse(uwh.getHosts().contains("other_host")); + } + + @Test public void testMultipleExplicitHostsNewConfig() { String server = "server", host1 = "host1", host2 = "host2", host3 = "host3"; setValidHostsNewConfig(server, Joiner.on(',').join(host1, host2, host3)); @@ -193,6 +322,33 @@ public class UserImpersonationTest { } @Test + public void testMultipleExplicitUsersHosts() { + String server = "server", host1 = "host1", host2 = "host2", host3 = "host3", client1 = "client1", client2 = "client2", client3 = "client3"; + setValidHosts(server, Joiner.on(',').join(host1, host2, host3)); + setValidUsers(server, Joiner.on(',').join(client1, client2, client3)); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server); + assertNotNull(uwh); + + assertFalse(uwh.acceptsAllHosts()); + assertFalse(uwh.acceptsAllUsers()); + + assertNotEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertNotEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + + assertTrue(uwh.getUsers().contains(client1)); + assertTrue(uwh.getUsers().contains(client2)); + assertTrue(uwh.getUsers().contains(client3)); + assertFalse(uwh.getUsers().contains("other_client")); + + assertTrue(uwh.getHosts().contains(host1)); + assertTrue(uwh.getHosts().contains(host2)); + assertTrue(uwh.getHosts().contains(host3)); + assertFalse(uwh.getHosts().contains("other_host")); + } + + @Test public void testMultipleExplicitUsersHostsNewConfig() { String server = "server", host1 = "host1", host2 = "host2", host3 = "host3", client1 = "client1", client2 = "client2", client3 = "client3"; setValidHostsNewConfig(server, Joiner.on(',').join(host1, host2, host3)); @@ -220,6 +376,59 @@ public class UserImpersonationTest { } @Test + public void testMultipleAllowedImpersonators() { + String server1 = "server1", server2 = "server2", host1 = "host1", host2 = "host2", host3 = "host3", client1 = "client1", client2 = "client2", client3 = "client3"; + // server1 can impersonate client1 and client2 from host1 or host2 + setValidHosts(server1, Joiner.on(',').join(host1, host2)); + setValidUsers(server1, Joiner.on(',').join(client1, client2)); + // server2 can impersonate only client3 from host3 + setValidHosts(server2, host3); + setValidUsers(server2, client3); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server1); + assertNotNull(uwh); + + assertFalse(uwh.acceptsAllHosts()); + assertFalse(uwh.acceptsAllUsers()); + + assertNotEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertNotEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + + assertTrue(uwh.getUsers().contains(client1)); + assertTrue(uwh.getUsers().contains(client2)); + assertFalse(uwh.getUsers().contains(client3)); + assertFalse(uwh.getUsers().contains("other_client")); + + assertTrue(uwh.getHosts().contains(host1)); + assertTrue(uwh.getHosts().contains(host2)); + assertFalse(uwh.getHosts().contains(host3)); + assertFalse(uwh.getHosts().contains("other_host")); + + uwh = impersonation.get(server2); + assertNotNull(uwh); + + assertFalse(uwh.acceptsAllHosts()); + assertFalse(uwh.acceptsAllUsers()); + + assertNotEquals(AlwaysTrueSet.class, uwh.getHosts().getClass()); + assertNotEquals(AlwaysTrueSet.class, uwh.getUsers().getClass()); + + assertFalse(uwh.getUsers().contains(client1)); + assertFalse(uwh.getUsers().contains(client2)); + assertTrue(uwh.getUsers().contains(client3)); + assertFalse(uwh.getUsers().contains("other_client")); + + assertFalse(uwh.getHosts().contains(host1)); + assertFalse(uwh.getHosts().contains(host2)); + assertTrue(uwh.getHosts().contains(host3)); + assertFalse(uwh.getHosts().contains("other_host")); + + // client3 is not allowed to impersonate anyone + assertNull(impersonation.get(client3)); + } + + @Test public void testMultipleAllowedImpersonatorsNewConfig() { String server1 = "server1", server2 = "server2", host1 = "host1", host2 = "host2", host3 = "host3", client1 = "client1", client2 = "client2", client3 = "client3"; // server1 can impersonate client1 and client2 from host1 or host2 @@ -270,6 +479,24 @@ public class UserImpersonationTest { assertNull(impersonation.get(client3)); } + @SuppressWarnings("deprecation") + @Test + public void testSingleUser() throws Exception { + final String server = "server/hostn...@example.com", client = "cli...@example.com"; + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + server + ".users", client); + cc.set(Property.INSTANCE_RPC_SASL_PROXYUSERS.getKey() + server + ".hosts", "*"); + UserImpersonation impersonation = new UserImpersonation(conf); + + UsersWithHosts uwh = impersonation.get(server); + + assertNotNull(uwh); + + assertTrue(uwh.acceptsAllHosts()); + assertFalse(uwh.acceptsAllUsers()); + + assertTrue(uwh.getUsers().contains(client)); + } + @Test public void testSingleUserNewConfig() throws Exception { final String server = "server/hostn...@example.com", client = "cli...@example.com"; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/base/src/test/java/org/apache/accumulo/server/util/TServerUtilsTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/TServerUtilsTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/TServerUtilsTest.java index fde9e6d..37d127a 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/TServerUtilsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/TServerUtilsTest.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.ServerSocket; import java.net.UnknownHostException; +import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.ExecutorService; @@ -87,6 +88,34 @@ public class TServerUtilsTest { return 30; } + @Deprecated + @Override + public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public AccumuloConfiguration getConfiguration() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public void setConfiguration(AccumuloConfiguration conf) {} + @Override public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException { throw new UnsupportedOperationException(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/master/src/main/java/org/apache/accumulo/master/Master.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java index 8233673..dbf0f87 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@ -343,7 +343,8 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP); } - String zpath = zooRoot + Constants.ZCONFIG + "/tserver.wal.sync.method"; + @SuppressWarnings("deprecation") + String zpath = zooRoot + Constants.ZCONFIG + "/" + Property.TSERV_WAL_SYNC_METHOD.getKey(); // is the entire instance set to use flushing vs sync? boolean flushDefault = false; try { @@ -357,7 +358,8 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) { log.debug("Converting table " + id + " WALog setting to Durability"); try { - String path = zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_CONF + "/table.walog.enabled"; + @SuppressWarnings("deprecation") + String path = zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_CONF + "/" + Property.TABLE_WALOG_ENABLED.getKey(); byte[] data = zoo.getData(path, null); boolean useWAL = Boolean.parseBoolean(new String(data, UTF_8)); zoo.recursiveDelete(path, NodeMissingPolicy.FAIL); http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/master/src/main/java/org/apache/accumulo/master/util/FateAdmin.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/util/FateAdmin.java b/server/master/src/main/java/org/apache/accumulo/master/util/FateAdmin.java new file mode 100644 index 0000000..a1dd303 --- /dev/null +++ b/server/master/src/main/java/org/apache/accumulo/master/util/FateAdmin.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.master.util; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.cli.Help; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.zookeeper.ZooUtil; +import org.apache.accumulo.fate.AdminUtil; +import org.apache.accumulo.fate.ReadOnlyStore; +import org.apache.accumulo.fate.ZooStore; +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; +import org.apache.accumulo.master.Master; +import org.apache.accumulo.server.client.HdfsZooInstance; +import org.apache.accumulo.server.zookeeper.ZooReaderWriter; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +/** + * A utility to administer FATE operations + */ +public class FateAdmin { + + static class TxOpts { + @Parameter(description = "<txid>...", required = true) + List<String> txids = new ArrayList<>(); + } + + @Parameters(commandDescription = "Stop an existing FATE by transaction id") + static class FailOpts extends TxOpts {} + + @Parameters(commandDescription = "Delete an existing FATE by transaction id") + static class DeleteOpts extends TxOpts {} + + @Parameters(commandDescription = "List the existing FATE transactions") + static class PrintOpts {} + + public static void main(String[] args) throws Exception { + Help opts = new Help(); + JCommander jc = new JCommander(opts); + jc.setProgramName(FateAdmin.class.getName()); + LinkedHashMap<String,TxOpts> txOpts = new LinkedHashMap<>(2); + txOpts.put("fail", new FailOpts()); + txOpts.put("delete", new DeleteOpts()); + for (Entry<String,TxOpts> entry : txOpts.entrySet()) { + jc.addCommand(entry.getKey(), entry.getValue()); + } + jc.addCommand("print", new PrintOpts()); + jc.parse(args); + if (opts.help || jc.getParsedCommand() == null) { + jc.usage(); + System.exit(1); + } + + System.err + .printf("This tool has been deprecated%nFATE administration now available within 'accumulo shell'%n$ fate fail <txid>... | delete <txid>... | print [<txid>...]%n%n"); + + AdminUtil<Master> admin = new AdminUtil<>(); + + Instance instance = HdfsZooInstance.getInstance(); + String path = ZooUtil.getRoot(instance) + Constants.ZFATE; + String masterPath = ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK; + IZooReaderWriter zk = ZooReaderWriter.getInstance(); + ZooStore<Master> zs = new ZooStore<>(path, zk); + + if (jc.getParsedCommand().equals("fail")) { + for (String txid : txOpts.get(jc.getParsedCommand()).txids) { + if (!admin.prepFail(zs, zk, masterPath, txid)) { + System.exit(1); + } + } + } else if (jc.getParsedCommand().equals("delete")) { + for (String txid : txOpts.get(jc.getParsedCommand()).txids) { + if (!admin.prepDelete(zs, zk, masterPath, txid)) { + System.exit(1); + } + admin.deleteLocks(zs, zk, ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS, txid); + } + } else if (jc.getParsedCommand().equals("print")) { + admin.print(new ReadOnlyStore<>(zs), zk, ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS); + } + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java ---------------------------------------------------------------------- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java index 0aa9d72..31bea15 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java @@ -81,6 +81,7 @@ public class ShellServlet extends BasicServlet { // user attribute is null, check to see if username and password are passed as parameters user = req.getParameter("user"); String pass = req.getParameter("pass"); + String mock = req.getParameter("mock"); if (user == null || pass == null) { // username or password are null, re-authenticate sb.append(authenticationForm(req.getRequestURI(), CSRF_TOKEN)); @@ -88,7 +89,7 @@ public class ShellServlet extends BasicServlet { } try { // get a new shell for this user - ShellExecutionThread shellThread = new ShellExecutionThread(user, pass); + ShellExecutionThread shellThread = new ShellExecutionThread(user, pass, mock); service().submit(shellThread); userShells().put(session.getId(), shellThread); } catch (IOException e) { @@ -223,7 +224,8 @@ public class ShellServlet extends BasicServlet { private String authenticationForm(String requestURI, String csrfToken) { return "<div id='login'><form method=POST action='" + requestURI + "'>" - + "<table><tr><td>Username: </td><td><input type='text' name='user'></td></tr>" + + "<table><tr><td>Mock: </td><td><input type='checkbox' name='mock' value='mock'></td></tr>" + + "<tr><td>Username: </td><td><input type='text' name='user'></td></tr>" + "<tr><td>Password: </td><td><input type='password' name='pass'></td><td>" + "<input type='hidden' name='" + CSRF_KEY + "' value='" + csrfToken + "'/><input type='submit' value='Enter'></td></tr></table></form></div>"; } @@ -253,7 +255,7 @@ public class ShellServlet extends BasicServlet { private boolean done; private boolean readWait; - private ShellExecutionThread(String username, String password) throws IOException { + private ShellExecutionThread(String username, String password, String mock) throws IOException { this.done = false; this.cmd = null; this.cmdIndex = 0; @@ -262,7 +264,10 @@ public class ShellServlet extends BasicServlet { ConsoleReader reader = new ConsoleReader(this, output); this.shell = new Shell(reader); shell.setLogErrorsToConsole(); - if (shell.config("-u", username, "-p", password)) { + if (mock != null) { + if (shell.config("--fake", "-u", username, "-p", password)) + throw new IOException("mock shell config error"); + } else if (shell.config("-u", username, "-p", password)) { throw new IOException("shell config error"); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/NullScanner.java ---------------------------------------------------------------------- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/NullScanner.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/NullScanner.java index d96d23f..b91d454 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/NullScanner.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/NullScanner.java @@ -53,6 +53,16 @@ public class NullScanner implements Scanner { @Override public void clearScanIterators() {} + @Deprecated + @Override + public void setTimeOut(int timeOut) {} + + @Deprecated + @Override + public int getTimeOut() { + return 0; + } + @Override public void setRange(Range range) {} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/tracer/src/test/java/org/apache/accumulo/tracer/TracerTest.java ---------------------------------------------------------------------- diff --git a/server/tracer/src/test/java/org/apache/accumulo/tracer/TracerTest.java b/server/tracer/src/test/java/org/apache/accumulo/tracer/TracerTest.java index 19891a4..6dcf7a4 100644 --- a/server/tracer/src/test/java/org/apache/accumulo/tracer/TracerTest.java +++ b/server/tracer/src/test/java/org/apache/accumulo/tracer/TracerTest.java @@ -99,6 +99,7 @@ public class TracerTest { public void close() throws IOException {} } + @SuppressWarnings("deprecation") @Test public void testTrace() throws Exception { TestReceiver tracer = new TestReceiver(); @@ -114,7 +115,7 @@ public class TracerTest { assertFalse(Trace.isTracing()); Span start = Trace.on("testing"); - assertEquals(org.apache.htrace.Trace.currentSpan(), start.getScope().getSpan()); + assertEquals(Trace.currentTrace().getSpan(), start.getScope().getSpan()); assertTrue(Trace.isTracing()); Span span = Trace.start("shortest trace ever"); http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/tserver/src/main/java/org/apache/accumulo/tserver/TservConstraintEnv.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TservConstraintEnv.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TservConstraintEnv.java index 5a50e8f..fc371c9 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TservConstraintEnv.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TservConstraintEnv.java @@ -24,6 +24,7 @@ import org.apache.accumulo.core.constraints.Constraint.Environment; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.impl.KeyExtent; import org.apache.accumulo.core.security.AuthorizationContainer; +import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.server.security.SecurityOperation; @@ -31,6 +32,7 @@ public class TservConstraintEnv implements Environment { private final TCredentials credentials; private final SecurityOperation security; + private Authorizations auths; private KeyExtent ke; TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) { @@ -53,6 +55,18 @@ public class TservConstraintEnv implements Environment { } @Override + @Deprecated + public Authorizations getAuthorizations() { + if (auths == null) + try { + this.auths = security.getUserAuthorizations(credentials); + } catch (ThriftSecurityException e) { + throw new RuntimeException(e); + } + return auths; + } + + @Override public AuthorizationContainer getAuthorizationsContainer() { return new AuthorizationContainer() { @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java index a3d8782..2667b53 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LocalWALRecovery.java @@ -48,6 +48,7 @@ import com.google.common.annotations.VisibleForTesting; /** * This class will attempt to rewrite any local WALs to HDFS. */ +@SuppressWarnings("deprecation") public class LocalWALRecovery implements Runnable { private static final Logger log = LoggerFactory.getLogger(LocalWALRecovery.class); @@ -153,8 +154,8 @@ public class LocalWALRecovery implements Runnable { Path localWal = new Path(file.toURI()); FileSystem localFs = FileSystem.getLocal(fs.getConf()); - Reader reader = new SequenceFile.Reader(localFs.getConf(), SequenceFile.Reader.file(localWal.makeQualified(localWal.toUri(), - localFs.getWorkingDirectory()))); + Reader reader = new SequenceFile.Reader(localFs, localWal, localFs.getConf()); + // Reader reader = new SequenceFile.Reader(localFs.getConf(), SequenceFile.Reader.file(localWal)); Path tmp = new Path(options.destination + "/" + name + ".copy"); FSDataOutputStream writer = fs.create(tmp); while (reader.next(key, value)) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java index d9a4e0e..b65d5ce 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java @@ -138,7 +138,8 @@ public class SortedLogRecoveryTest { for (Entry<String,KeyValue[]> entry : logs.entrySet()) { String path = workdir + "/" + entry.getKey(); FileSystem ns = fs.getVolumeByPath(new Path(path)).getFileSystem(); - Writer map = new MapFile.Writer(ns.getConf(), new Path(path + "/log1"), Writer.keyClass(LogFileKey.class), Writer.valueClass(LogFileValue.class)); + @SuppressWarnings("deprecation") + Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class); for (KeyValue lfe : entry.getValue()) { map.append(lfe.key, lfe.value); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java index 7a1a785..d9c6862 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java @@ -41,10 +41,6 @@ import org.junit.rules.TemporaryFolder; * */ public class RootFilesTest { - @SuppressWarnings("deprecation") - private static final Property INSTANCE_DFS_DIR = Property.INSTANCE_DFS_DIR; - @SuppressWarnings("deprecation") - private static final Property INSTANCE_DFS_URI = Property.INSTANCE_DFS_URI; @Rule public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); @@ -120,12 +116,13 @@ public class RootFilesTest { } } + @SuppressWarnings("deprecation") @Test public void testFileReplacement() throws IOException { ConfigurationCopy conf = new ConfigurationCopy(); - conf.set(INSTANCE_DFS_URI, "file:///"); - conf.set(INSTANCE_DFS_DIR, "/"); + conf.set(Property.INSTANCE_DFS_URI, "file:///"); + conf.set(Property.INSTANCE_DFS_DIR, "/"); VolumeManager vm = VolumeManagerImpl.get(conf); http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/main/java/org/apache/accumulo/shell/Shell.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/accumulo/shell/Shell.java b/shell/src/main/java/org/apache/accumulo/shell/Shell.java index 1819f43..7678ead 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java +++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java @@ -66,6 +66,7 @@ import org.apache.accumulo.core.data.thrift.TConstraintViolationSummary; import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException; import org.apache.accumulo.core.trace.DistributedTrace; import org.apache.accumulo.core.util.BadArgumentException; +import org.apache.accumulo.core.util.DeprecationUtil; import org.apache.accumulo.core.util.format.DefaultFormatter; import org.apache.accumulo.core.util.format.Formatter; import org.apache.accumulo.core.util.format.FormatterConfig; @@ -95,6 +96,7 @@ import org.apache.accumulo.shell.commands.DeleteIterCommand; import org.apache.accumulo.shell.commands.DeleteManyCommand; import org.apache.accumulo.shell.commands.DeleteNamespaceCommand; import org.apache.accumulo.shell.commands.DeleteRowsCommand; +import org.apache.accumulo.shell.commands.DeleteScanIterCommand; import org.apache.accumulo.shell.commands.DeleteShellIterCommand; import org.apache.accumulo.shell.commands.DeleteTableCommand; import org.apache.accumulo.shell.commands.DeleteUserCommand; @@ -147,6 +149,7 @@ import org.apache.accumulo.shell.commands.ScriptCommand; import org.apache.accumulo.shell.commands.SetAuthsCommand; import org.apache.accumulo.shell.commands.SetGroupsCommand; import org.apache.accumulo.shell.commands.SetIterCommand; +import org.apache.accumulo.shell.commands.SetScanIterCommand; import org.apache.accumulo.shell.commands.SetShellIterCommand; import org.apache.accumulo.shell.commands.SleepCommand; import org.apache.accumulo.shell.commands.SystemPermissionsCommand; @@ -368,7 +371,9 @@ public class Shell extends ShellOptions implements KeywordExecutable { } } - DistributedTrace.enable(InetAddress.getLocalHost().getHostName(), "shell", clientConf); + if (!options.isFake()) { + DistributedTrace.enable(InetAddress.getLocalHost().getHostName(), "shell", clientConf); + } this.setTableName(""); connector = instance.getConnector(user, token); @@ -401,8 +406,8 @@ public class Shell extends ShellOptions implements KeywordExecutable { Command[] execCommands = {new ExecfileCommand(), new HistoryCommand(), new ExtensionCommand(), new ScriptCommand()}; Command[] exitCommands = {new ByeCommand(), new ExitCommand(), new QuitCommand()}; Command[] helpCommands = {new AboutCommand(), new HelpCommand(), new InfoCommand(), new QuestionCommand()}; - Command[] iteratorCommands = {new DeleteIterCommand(), new ListIterCommand(), new SetIterCommand(), new SetShellIterCommand(), new ListShellIterCommand(), - new DeleteShellIterCommand()}; + Command[] iteratorCommands = {new DeleteIterCommand(), new DeleteScanIterCommand(), new ListIterCommand(), new SetIterCommand(), new SetScanIterCommand(), + new SetShellIterCommand(), new ListShellIterCommand(), new DeleteShellIterCommand()}; Command[] otherCommands = {new HiddenCommand()}; Command[] permissionsCommands = {new GrantCommand(), new RevokeCommand(), new SystemPermissionsCommand(), new TablePermissionsCommand(), new UserPermissionsCommand(), new NamespacePermissionsCommand()}; @@ -446,24 +451,28 @@ public class Shell extends ShellOptions implements KeywordExecutable { protected void setInstance(ShellOptionsJC options) { // should only be one set of instance options set instance = null; - String instanceName, hosts; - if (options.isHdfsZooInstance()) { - instanceName = hosts = null; - } else if (options.getZooKeeperInstance().size() > 0) { - List<String> zkOpts = options.getZooKeeperInstance(); - instanceName = zkOpts.get(0); - hosts = zkOpts.get(1); + if (options.isFake()) { + instance = DeprecationUtil.makeMockInstance("fake"); } else { - instanceName = options.getZooKeeperInstanceName(); - hosts = options.getZooKeeperHosts(); - } - final ClientConfiguration clientConf; - try { - clientConf = options.getClientConfiguration(); - } catch (ConfigurationException | FileNotFoundException e) { - throw new IllegalArgumentException("Unable to load client config from " + options.getClientConfigFile(), e); + String instanceName, hosts; + if (options.isHdfsZooInstance()) { + instanceName = hosts = null; + } else if (options.getZooKeeperInstance().size() > 0) { + List<String> zkOpts = options.getZooKeeperInstance(); + instanceName = zkOpts.get(0); + hosts = zkOpts.get(1); + } else { + instanceName = options.getZooKeeperInstanceName(); + hosts = options.getZooKeeperHosts(); + } + final ClientConfiguration clientConf; + try { + clientConf = options.getClientConfiguration(); + } catch (ConfigurationException | FileNotFoundException e) { + throw new IllegalArgumentException("Unable to load client config from " + options.getClientConfigFile(), e); + } + instance = getZooInstance(instanceName, hosts, clientConf); } - instance = getZooInstance(instanceName, hosts, clientConf); } /** http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java b/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java index 7f476c1..1e5156f 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java +++ b/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java @@ -241,6 +241,10 @@ public class ShellOptionsJC { return debugEnabled; } + public boolean isFake() { + return fake; + } + public boolean isHelpEnabled() { return helpEnabled; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/main/java/org/apache/accumulo/shell/commands/DeleteScanIterCommand.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/DeleteScanIterCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/DeleteScanIterCommand.java new file mode 100644 index 0000000..7c8cf22 --- /dev/null +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/DeleteScanIterCommand.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.shell.commands; + +import java.util.Iterator; +import java.util.List; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.shell.Shell; +import org.apache.accumulo.shell.Shell.Command; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; + +public class DeleteScanIterCommand extends Command { + private Option nameOpt, allOpt; + + @Override + public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception { + Shell.log.warn("Deprecated, use " + new DeleteShellIterCommand().getName()); + final String tableName = OptUtil.getTableOpt(cl, shellState); + + if (cl.hasOption(allOpt.getOpt())) { + final List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.remove(tableName); + if (tableScanIterators == null) { + Shell.log.info("No scan iterators set on table " + tableName); + } else { + Shell.log.info("Removed the following scan iterators from table " + tableName + ":" + tableScanIterators); + } + } else if (cl.hasOption(nameOpt.getOpt())) { + final String name = cl.getOptionValue(nameOpt.getOpt()); + final List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.get(tableName); + if (tableScanIterators != null) { + boolean found = false; + for (Iterator<IteratorSetting> iter = tableScanIterators.iterator(); iter.hasNext();) { + if (iter.next().getName().equals(name)) { + iter.remove(); + found = true; + break; + } + } + if (!found) { + Shell.log.info("No iterator named " + name + " found for table " + tableName); + } else { + Shell.log.info("Removed scan iterator " + name + " from table " + tableName + " (" + shellState.scanIteratorOptions.get(tableName).size() + " left)"); + if (shellState.scanIteratorOptions.get(tableName).size() == 0) { + shellState.scanIteratorOptions.remove(tableName); + } + } + } else { + Shell.log.info("No iterator named " + name + " found for table " + tableName); + } + } + + return 0; + } + + @Override + public String description() { + return "deletes a table-specific scan iterator so it is no longer used during this shell session"; + } + + @Override + public Options getOptions() { + final Options o = new Options(); + + OptionGroup nameGroup = new OptionGroup(); + + nameOpt = new Option("n", "name", true, "iterator to delete"); + nameOpt.setArgName("itername"); + + allOpt = new Option("a", "all", false, "delete all scan iterators"); + allOpt.setArgName("all"); + + nameGroup.addOption(nameOpt); + nameGroup.addOption(allOpt); + nameGroup.setRequired(true); + o.addOptionGroup(nameGroup); + o.addOption(OptUtil.tableOpt("table to delete scan iterators from")); + + return o; + } + + @Override + public int numArgs() { + return 0; + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/main/java/org/apache/accumulo/shell/commands/SetIterCommand.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/SetIterCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/SetIterCommand.java index 6e89298..fffdf21 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/SetIterCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/SetIterCommand.java @@ -53,7 +53,7 @@ import jline.console.ConsoleReader; public class SetIterCommand extends Command { private Option allScopeOpt, mincScopeOpt, majcScopeOpt, scanScopeOpt, nameOpt, priorityOpt; - private Option ageoffTypeOpt, regexTypeOpt, versionTypeOpt, reqvisTypeOpt, classnameTypeOpt; + private Option aggTypeOpt, ageoffTypeOpt, regexTypeOpt, versionTypeOpt, reqvisTypeOpt, classnameTypeOpt; @Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, @@ -66,7 +66,12 @@ public class SetIterCommand extends Command { final Map<String,String> options = new HashMap<>(); String classname = cl.getOptionValue(classnameTypeOpt.getOpt()); - if (cl.hasOption(regexTypeOpt.getOpt())) { + if (cl.hasOption(aggTypeOpt.getOpt())) { + Shell.log.warn("aggregators are deprecated"); + @SuppressWarnings("deprecation") + String deprecatedClassName = org.apache.accumulo.core.iterators.AggregatingIterator.class.getName(); + classname = deprecatedClassName; + } else if (cl.hasOption(regexTypeOpt.getOpt())) { classname = RegExFilter.class.getName(); } else if (cl.hasOption(ageoffTypeOpt.getOpt())) { classname = AgeOffFilter.class.getName(); @@ -114,6 +119,14 @@ public class SetIterCommand extends Command { ScanCommand.ensureTserversCanLoadIterator(shellState, tableName, classname); + final String aggregatorClass = options.get("aggregatorClass"); + @SuppressWarnings("deprecation") + String deprecatedAggregatorClassName = org.apache.accumulo.core.iterators.aggregation.Aggregator.class.getName(); + if (aggregatorClass != null && !shellState.getConnector().tableOperations().testClassLoad(tableName, aggregatorClass, deprecatedAggregatorClassName)) { + throw new ShellCommandException(ErrorCode.INITIALIZATION_FAILURE, "Servers are unable to load " + aggregatorClass + " as type " + + deprecatedAggregatorClassName); + } + for (Iterator<Entry<String,String>> i = options.entrySet().iterator(); i.hasNext();) { final Entry<String,String> entry = i.next(); if (entry.getValue() == null || entry.getValue().isEmpty()) { @@ -148,6 +161,14 @@ public class SetIterCommand extends Command { + SortedKeyValueIterator.class.getName()); } + final String aggregatorClass = options.get("aggregatorClass"); + @SuppressWarnings("deprecation") + String deprecatedAggregatorClassName = org.apache.accumulo.core.iterators.aggregation.Aggregator.class.getName(); + if (aggregatorClass != null && !shellState.getConnector().namespaceOperations().testClassLoad(namespace, aggregatorClass, deprecatedAggregatorClassName)) { + throw new ShellCommandException(ErrorCode.INITIALIZATION_FAILURE, "Servers are unable to load " + aggregatorClass + " as type " + + deprecatedAggregatorClassName); + } + for (Iterator<Entry<String,String>> i = options.entrySet().iterator(); i.hasNext();) { final Entry<String,String> entry = i.next(); if (entry.getValue() == null || entry.getValue().isEmpty()) { @@ -336,12 +357,14 @@ public class SetIterCommand extends Command { final OptionGroup typeGroup = new OptionGroup(); classnameTypeOpt = new Option("class", "class-name", true, "a java class that implements SortedKeyValueIterator"); classnameTypeOpt.setArgName("name"); + aggTypeOpt = new Option("agg", "aggregator", false, "an aggregating type"); regexTypeOpt = new Option("regex", "regular-expression", false, "a regex matching iterator"); versionTypeOpt = new Option("vers", "version", false, "a versioning iterator"); reqvisTypeOpt = new Option("reqvis", "require-visibility", false, "an iterator that omits entries with empty visibilities"); ageoffTypeOpt = new Option("ageoff", "ageoff", false, "an aging off iterator"); typeGroup.addOption(classnameTypeOpt); + typeGroup.addOption(aggTypeOpt); typeGroup.addOption(regexTypeOpt); typeGroup.addOption(versionTypeOpt); typeGroup.addOption(reqvisTypeOpt); http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/main/java/org/apache/accumulo/shell/commands/SetScanIterCommand.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/SetScanIterCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/SetScanIterCommand.java new file mode 100644 index 0000000..2399d0e --- /dev/null +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/SetScanIterCommand.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.shell.commands; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.shell.Shell; +import org.apache.accumulo.shell.ShellCommandException; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; + +public class SetScanIterCommand extends SetIterCommand { + @Override + public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, + TableNotFoundException, IOException, ShellCommandException { + Shell.log.warn("Deprecated, use " + new SetShellIterCommand().getName()); + return super.execute(fullCommand, cl, shellState); + } + + @Override + protected void setTableProperties(final CommandLine cl, final Shell shellState, final int priority, final Map<String,String> options, final String classname, + final String name) throws AccumuloException, AccumuloSecurityException, ShellCommandException, TableNotFoundException { + + final String tableName = OptUtil.getTableOpt(cl, shellState); + + ScanCommand.ensureTserversCanLoadIterator(shellState, tableName, classname); + + for (Iterator<Entry<String,String>> i = options.entrySet().iterator(); i.hasNext();) { + final Entry<String,String> entry = i.next(); + if (entry.getValue() == null || entry.getValue().isEmpty()) { + i.remove(); + } + } + + List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.get(tableName); + if (tableScanIterators == null) { + tableScanIterators = new ArrayList<>(); + shellState.scanIteratorOptions.put(tableName, tableScanIterators); + } + final IteratorSetting setting = new IteratorSetting(priority, name, classname); + setting.addOptions(options); + + // initialize a scanner to ensure the new setting does not conflict with existing settings + final String user = shellState.getConnector().whoami(); + final Authorizations auths = shellState.getConnector().securityOperations().getUserAuthorizations(user); + final Scanner scanner = shellState.getConnector().createScanner(tableName, auths); + for (IteratorSetting s : tableScanIterators) { + scanner.addScanIterator(s); + } + scanner.addScanIterator(setting); + + // if no exception has been thrown, it's safe to add it to the list + tableScanIterators.add(setting); + Shell.log.debug("Scan iterators :" + shellState.scanIteratorOptions.get(tableName)); + } + + @Override + public String description() { + return "sets a table-specific scan iterator for this shell session"; + } + + @Override + public Options getOptions() { + // Remove the options that specify which type of iterator this is, since + // they are all scan iterators with this command. + final HashSet<OptionGroup> groups = new HashSet<>(); + final Options parentOptions = super.getOptions(); + final Options modifiedOptions = new Options(); + for (Iterator<?> it = parentOptions.getOptions().iterator(); it.hasNext();) { + Option o = (Option) it.next(); + if (!IteratorScope.majc.name().equals(o.getOpt()) && !IteratorScope.minc.name().equals(o.getOpt()) && !IteratorScope.scan.name().equals(o.getOpt())) { + modifiedOptions.addOption(o); + OptionGroup group = parentOptions.getOptionGroup(o); + if (group != null) + groups.add(group); + } + } + for (OptionGroup group : groups) { + modifiedOptions.addOptionGroup(group); + } + return modifiedOptions; + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java b/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java new file mode 100644 index 0000000..ebc92f7 --- /dev/null +++ b/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.shell.mock; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import jline.console.ConsoleReader; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.shell.Shell; +import org.apache.accumulo.shell.ShellOptionsJC; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.vfs2.FileSystemException; + +/** + * An Accumulo Shell implementation that allows a developer to attach an InputStream and Writer to the Shell for testing purposes. + * + * @deprecated since 1.8.0; use MiniAccumuloCluster or a standard mock framework instead. + */ +@Deprecated +public class MockShell extends Shell { + private static final String NEWLINE = "\n"; + + protected InputStream in; + protected OutputStream out; + + public MockShell(InputStream in, OutputStream out) throws IOException { + super(); + this.in = in; + this.out = out; + } + + @Override + public boolean config(String... args) throws IOException { + // If configuring the shell failed, fail quickly + if (!super.config(args)) { + return false; + } + + // Update the ConsoleReader with the input and output "redirected" + try { + this.reader = new ConsoleReader(in, out); + } catch (Exception e) { + printException(e); + return false; + } + + // Don't need this for testing purposes + this.reader.setHistoryEnabled(false); + this.reader.setPaginationEnabled(false); + + // Make the parsing from the client easier; + this.verbose = false; + return true; + } + + @Override + protected void setInstance(ShellOptionsJC options) { + // We always want a MockInstance for this test + instance = new org.apache.accumulo.core.client.mock.MockInstance(); + } + + @Override + public int start() throws IOException { + String input; + if (isVerbose()) + printInfo(); + + if (execFile != null) { + java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name()); + try { + while (scanner.hasNextLine() && !hasExited()) { + execCommand(scanner.nextLine(), true, isVerbose()); + } + } finally { + scanner.close(); + } + } else if (execCommand != null) { + for (String command : execCommand.split("\n")) { + execCommand(command, true, isVerbose()); + } + return exitCode; + } + + while (true) { + if (hasExited()) + return exitCode; + + reader.setPrompt(getDefaultPrompt()); + input = reader.readLine(); + if (input == null) { + reader.println(); + return exitCode; + } // user canceled + + execCommand(input, false, false); + } + } + + /** + * @param in + * the in to set + */ + public void setConsoleInputStream(InputStream in) { + this.in = in; + } + + /** + * @param out + * the output stream to set + */ + public void setConsoleWriter(OutputStream out) { + this.out = out; + } + + @Override + public ClassLoader getClassLoader(final CommandLine cl, final Shell shellState) throws AccumuloException, TableNotFoundException, AccumuloSecurityException, + IOException, FileSystemException { + return MockShell.class.getClassLoader(); + } + + /** + * Convenience method to create the byte-array to hand to the console + * + * @param commands + * An array of commands to run + * @return A byte[] input stream which can be handed to the console. + */ + public static ByteArrayInputStream makeCommands(String... commands) { + StringBuilder sb = new StringBuilder(commands.length * 8); + + for (String command : commands) { + sb.append(command).append(NEWLINE); + } + + return new ByteArrayInputStream(sb.toString().getBytes(UTF_8)); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/test/java/org/apache/accumulo/shell/ShellConfigTest.java ---------------------------------------------------------------------- diff --git a/shell/src/test/java/org/apache/accumulo/shell/ShellConfigTest.java b/shell/src/test/java/org/apache/accumulo/shell/ShellConfigTest.java index 5e8736c..8bef14d 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/ShellConfigTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/ShellConfigTest.java @@ -24,48 +24,29 @@ import java.io.File; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.IOException; -import java.io.OutputStream; import java.io.PrintStream; import java.nio.file.Files; import java.util.HashMap; import java.util.Map; +import jline.console.ConsoleReader; + import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.shell.ShellTest.TestOutputStream; import org.apache.log4j.Level; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.beust.jcommander.ParameterException; -import jline.console.ConsoleReader; - public class ShellConfigTest { - - public static class TestOutputStream extends OutputStream { - StringBuilder sb = new StringBuilder(); - - @Override - public void write(int b) throws IOException { - sb.append((char) (0xff & b)); - } - - public String get() { - return sb.toString(); - } - - public void clear() { - sb.setLength(0); - } - } - TestOutputStream output; Shell shell; PrintStream out; @@ -121,20 +102,17 @@ public class ShellConfigTest { assertTrue("Did not print usage", output.get().startsWith("Usage")); } - @Ignore @Test public void testTokenWithoutOptions() throws IOException { assertFalse(shell.config(args("--fake", "-tc", PasswordToken.class.getName()))); assertFalse(output.get().contains(ParameterException.class.getName())); } - @Ignore @Test public void testTokenAndOption() throws IOException { assertTrue(shell.config(args("--fake", "-tc", PasswordToken.class.getName(), "-u", "foo", "-l", "password=foo"))); } - @Ignore @Test public void testTokenAndOptionAndPassword() throws IOException { assertFalse(shell.config(args("--fake", "-tc", PasswordToken.class.getName(), "-l", "password=foo", "-p", "bar"))); http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java ---------------------------------------------------------------------- diff --git a/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java b/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java index 6dae0b4..428481a 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java @@ -34,6 +34,8 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +import jline.console.ConsoleReader; + import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; import org.apache.accumulo.core.client.ZooKeeperInstance; @@ -47,6 +49,7 @@ import org.apache.log4j.Level; import org.easymock.EasyMock; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; @@ -56,17 +59,10 @@ import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import jline.console.ConsoleReader; - @RunWith(PowerMockRunner.class) @PowerMockIgnore("javax.security.*") @PrepareForTest({Shell.class, ZooUtil.class, ConfigSanityCheck.class}) public class ShellSetInstanceTest { - @SuppressWarnings("deprecation") - private static Property INSTANCE_DFS_DIR = Property.INSTANCE_DFS_DIR; - @SuppressWarnings("deprecation") - private static Property INSTANCE_DFS_URI = Property.INSTANCE_DFS_URI; - public static class TestOutputStream extends OutputStream { StringBuilder sb = new StringBuilder(); @@ -125,6 +121,17 @@ public class ShellSetInstanceTest { SiteConfiguration.clearInstance(); } + @Deprecated + @Test + public void testSetInstance_Fake() throws Exception { + ShellOptionsJC opts = createMock(ShellOptionsJC.class); + expect(opts.isFake()).andReturn(true); + replay(opts); + + shell.setInstance(opts); + Assert.assertTrue(shell.getInstance() instanceof org.apache.accumulo.core.client.mock.MockInstance); + } + @Test public void testSetInstance_HdfsZooInstance_Explicit() throws Exception { testSetInstance_HdfsZooInstance(true, false, false); @@ -148,6 +155,7 @@ public class ShellSetInstanceTest { private void testSetInstance_HdfsZooInstance(boolean explicitHdfs, boolean onlyInstance, boolean onlyHosts) throws Exception { ClientConfiguration clientConf = createMock(ClientConfiguration.class); ShellOptionsJC opts = createMock(ShellOptionsJC.class); + expect(opts.isFake()).andReturn(false); expect(opts.getClientConfiguration()).andReturn(clientConf); expect(opts.isHdfsZooInstance()).andReturn(explicitHdfs); if (!explicitHdfs) { @@ -183,10 +191,14 @@ public class ShellSetInstanceTest { } if (!onlyInstance) { expect(clientConf.containsKey(Property.INSTANCE_VOLUMES.getKey())).andReturn(false).atLeastOnce(); - expect(clientConf.containsKey(INSTANCE_DFS_DIR.getKey())).andReturn(true).atLeastOnce(); - expect(clientConf.containsKey(INSTANCE_DFS_URI.getKey())).andReturn(true).atLeastOnce(); - expect(clientConf.getString(INSTANCE_DFS_URI.getKey())).andReturn("hdfs://nn1").atLeastOnce(); - expect(clientConf.getString(INSTANCE_DFS_DIR.getKey())).andReturn("/dfs").atLeastOnce(); + @SuppressWarnings("deprecation") + String INSTANCE_DFS_DIR_KEY = Property.INSTANCE_DFS_DIR.getKey(); + @SuppressWarnings("deprecation") + String INSTANCE_DFS_URI_KEY = Property.INSTANCE_DFS_URI.getKey(); + expect(clientConf.containsKey(INSTANCE_DFS_DIR_KEY)).andReturn(true).atLeastOnce(); + expect(clientConf.containsKey(INSTANCE_DFS_URI_KEY)).andReturn(true).atLeastOnce(); + expect(clientConf.getString(INSTANCE_DFS_URI_KEY)).andReturn("hdfs://nn1").atLeastOnce(); + expect(clientConf.getString(INSTANCE_DFS_DIR_KEY)).andReturn("/dfs").atLeastOnce(); } UUID randomUUID = null; @@ -221,6 +233,7 @@ public class ShellSetInstanceTest { private void testSetInstance_ZKInstance(boolean dashZ) throws Exception { ClientConfiguration clientConf = createMock(ClientConfiguration.class); ShellOptionsJC opts = createMock(ShellOptionsJC.class); + expect(opts.isFake()).andReturn(false); expect(opts.getClientConfiguration()).andReturn(clientConf); expect(opts.isHdfsZooInstance()).andReturn(false); expect(clientConf.getKeys()).andReturn(Arrays.asList(ClientProperty.INSTANCE_NAME.getKey(), ClientProperty.INSTANCE_ZK_HOST.getKey()).iterator()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f2e6472/shell/src/test/java/org/apache/accumulo/shell/ShellTest.java ---------------------------------------------------------------------- diff --git a/shell/src/test/java/org/apache/accumulo/shell/ShellTest.java b/shell/src/test/java/org/apache/accumulo/shell/ShellTest.java new file mode 100644 index 0000000..dc902ce --- /dev/null +++ b/shell/src/test/java/org/apache/accumulo/shell/ShellTest.java @@ -0,0 +1,394 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.shell; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +import org.apache.log4j.Level; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jline.console.ConsoleReader; + +public class ShellTest { + private static final Logger log = LoggerFactory.getLogger(ShellTest.class); + + public static class TestOutputStream extends OutputStream { + StringBuilder sb = new StringBuilder(); + + @Override + public void write(int b) throws IOException { + sb.append((char) (0xff & b)); + } + + public String get() { + return sb.toString(); + } + + public void clear() { + sb.setLength(0); + } + } + + public static class StringInputStream extends InputStream { + private String source = ""; + private int offset = 0; + + @Override + public int read() throws IOException { + if (offset == source.length()) + return '\n'; + else + return source.charAt(offset++); + } + + public void set(String other) { + source = other; + offset = 0; + } + } + + private StringInputStream input; + private TestOutputStream output; + private Shell shell; + private File config; + + void execExpectList(String cmd, boolean expecteGoodExit, List<String> expectedStrings) throws IOException { + exec(cmd); + if (expecteGoodExit) { + assertGoodExit("", true); + } else { + assertBadExit("", true); + } + + for (String expectedString : expectedStrings) { + assertTrue(expectedString + " was not present in " + output.get(), output.get().contains(expectedString)); + } + } + + void exec(String cmd) throws IOException { + output.clear(); + shell.execCommand(cmd, true, true); + } + + void exec(String cmd, boolean expectGoodExit) throws IOException { + exec(cmd); + if (expectGoodExit) + assertGoodExit("", true); + else + assertBadExit("", true); + } + + void exec(String cmd, boolean expectGoodExit, String expectString) throws IOException { + exec(cmd, expectGoodExit, expectString, true); + } + + void exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException { + exec(cmd); + if (expectGoodExit) + assertGoodExit(expectString, stringPresent); + else + assertBadExit(expectString, stringPresent); + } + + @Before + public void setup() throws IOException { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + Shell.log.setLevel(Level.OFF); + output = new TestOutputStream(); + input = new StringInputStream(); + config = Files.createTempFile(null, null).toFile(); + shell = new Shell(new ConsoleReader(input, output)); + shell.setLogErrorsToConsole(); + shell.config("--config-file", config.toString(), "--fake", "-u", "test", "-p", "secret"); + } + + @After + public void teardown() { + if (config.exists()) { + if (!config.delete()) { + log.error("Unable to delete {}", config); + } + } + shell.shutdown(); + } + + void assertGoodExit(String s, boolean stringPresent) { + Shell.log.debug(output.get()); + assertEquals(shell.getExitCode(), 0); + if (s.length() > 0) + assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s)); + } + + void assertBadExit(String s, boolean stringPresent) { + Shell.log.debug(output.get()); + assertTrue(shell.getExitCode() > 0); + if (s.length() > 0) + assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s)); + shell.resetExitCode(); + } + + @Test + public void aboutTest() throws IOException { + Shell.log.debug("Starting about test -----------------------------------"); + exec("about", true, "Shell - Apache Accumulo Interactive Shell"); + exec("about -v", true, "Current user:"); + exec("about arg", false, "java.lang.IllegalArgumentException: Expected 0 arguments"); + } + + @Test + public void addGetSplitsTest() throws IOException { + Shell.log.debug("Starting addGetSplits test ----------------------------"); + exec("addsplits arg", false, "java.lang.IllegalStateException: Not in a table context"); + exec("createtable test", true); + exec("addsplits 1 \\x80", true); + exec("getsplits", true, "1\n\\x80"); + exec("getsplits -m 1", true, "1"); + exec("getsplits -b64", true, "MQ==\ngA=="); + exec("deletetable test -f", true, "Table: [test] has been deleted"); + } + + @Test + public void insertDeleteScanTest() throws IOException { + Shell.log.debug("Starting insertDeleteScan test ------------------------"); + exec("insert r f q v", false, "java.lang.IllegalStateException: Not in a table context"); + exec("delete r f q", false, "java.lang.IllegalStateException: Not in a table context"); + exec("createtable test", true); + exec("insert r f q v", true); + exec("scan", true, "r f:q [] v"); + exec("delete r f q", true); + exec("scan", true, "r f:q [] v", false); + exec("insert \\x90 \\xa0 \\xb0 \\xc0\\xd0\\xe0\\xf0", true); + exec("scan", true, "\\x90 \\xA0:\\xB0 [] \\xC0\\xD0"); + exec("scan -f 2", true, "\\x90 \\xA0:\\xB0 [] \\xC0\\xD0"); + exec("scan -f 2", true, "\\x90 \\xA0:\\xB0 [] \\xC0\\xD0\\xE0", false); + exec("scan -b \\x90 -e \\x90 -c \\xA0", true, "\\x90 \\xA0:\\xB0 [] \\xC0"); + exec("scan -b \\x90 -e \\x90 -c \\xA0:\\xB0", true, "\\x90 \\xA0:\\xB0 [] \\xC0"); + exec("scan -b \\x90 -be", true, "\\x90 \\xA0:\\xB0 [] \\xC0", false); + exec("scan -e \\x90 -ee", true, "\\x90 \\xA0:\\xB0 [] \\xC0", false); + exec("scan -b \\x90\\x00", true, "\\x90 \\xA0:\\xB0 [] \\xC0", false); + exec("scan -e \\x8f", true, "\\x90 \\xA0:\\xB0 [] \\xC0", false); + exec("delete \\x90 \\xa0 \\xb0", true); + exec("scan", true, "\\x90 \\xA0:\\xB0 [] \\xC0", false); + exec("deletetable test -f", true, "Table: [test] has been deleted"); + } + + @Test + public void deleteManyTest() throws IOException { + exec("deletemany", false, "java.lang.IllegalStateException: Not in a table context"); + exec("createtable test", true); + exec("deletemany", true, "\n"); + + exec("insert 0 0 0 0 -ts 0"); + exec("insert 0 0 0 0 -l 0 -ts 0"); + exec("insert 1 1 1 1 -ts 1"); + exec("insert 2 2 2 2 -ts 2"); + + // prompts for delete, and rejects by default + exec("deletemany", true, "[SKIPPED] 0 0:0 []"); + exec("deletemany -r 0", true, "[SKIPPED] 0 0:0 []"); + exec("deletemany -r 0 -f", true, "[DELETED] 0 0:0 []"); + + // with auths, can delete the other record + exec("setauths -s 0"); + exec("deletemany -r 0 -f", true, "[DELETED] 0 0:0 [0]"); + + // delete will show the timestamp + exec("deletemany -r 1 -f -st", true, "[DELETED] 1 1:1 [] 1"); + + // DeleteManyCommand has its own Formatter (DeleterFormatter), so it does not honor the -fm flag + exec("deletemany -r 2 -f -st -fm org.apache.accumulo.core.util.format.DateStringFormatter", true, "[DELETED] 2 2:2 [] 2"); + + exec("setauths -c ", true); + exec("deletetable test -f", true, "Table: [test] has been deleted"); + } + + @Test + public void authsTest() throws Exception { + Shell.log.debug("Starting auths test --------------------------"); + exec("setauths x,y,z", false, "Missing required option"); + exec("setauths -s x,y,z -u notauser", false, "user does not exist"); + exec("setauths -s y,z,x", true); + exec("getauths -u notauser", false, "user does not exist"); + execExpectList("getauths", true, Arrays.asList("x", "y", "z")); + exec("addauths -u notauser", false, "Missing required option"); + exec("addauths -u notauser -s foo", false, "user does not exist"); + exec("addauths -s a", true); + execExpectList("getauths", true, Arrays.asList("x", "y", "z", "a")); + exec("setauths -c", true); + } + + @Test + public void userTest() throws Exception { + Shell.log.debug("Starting user test --------------------------"); + // Test cannot be done via junit because createuser only prompts for password + // exec("createuser root", false, "user exists"); + } + + @Test + public void duContextTest() throws Exception { + Shell.log.debug("Starting du context test --------------------------"); + exec("createtable t", true); + exec("du", true, "0 [t]"); + exec("deletetable t -f", true, "Table: [t] has been deleted"); + } + + @Test + public void duTest() throws IOException { + Shell.log.debug("Starting DU test --------------------------"); + exec("createtable t", true); + exec("du t", true, "0 [t]"); + exec("deletetable t -f", true, "Table: [t] has been deleted"); + } + + @Test + public void duPatternTest() throws IOException { + Shell.log.debug("Starting DU with pattern test --------------------------"); + exec("createtable t", true); + exec("createtable tt", true); + exec("du -p t.*", true, "0 [t, tt]"); + exec("deletetable t -f", true, "Table: [t] has been deleted"); + exec("deletetable tt -f", true, "Table: [tt] has been deleted"); + } + + @Test + public void scanTimestampTest() throws IOException { + Shell.log.debug("Starting scanTimestamp test ------------------------"); + exec("createtable test", true); + exec("insert r f q v -ts 0", true); + exec("scan -st", true, "r f:q [] 0 v"); + exec("scan -st -f 0", true, " : [] 0 "); + exec("deletemany -f", true); + exec("deletetable test -f", true, "Table: [test] has been deleted"); + } + + @Test + public void scanFewTest() throws IOException { + Shell.log.debug("Starting scanFew test ------------------------"); + exec("createtable test", true); + // historically, showing few did not pertain to ColVis or Timestamp + exec("insert 1 123 123456 -l '12345678' -ts 123456789 1234567890", true); + exec("setauths -s 12345678", true); + String expected = "1 123:123456 [12345678] 123456789 1234567890"; + String expectedFew = "1 123:12345 [12345678] 123456789 12345"; + exec("scan -st", true, expected); + exec("scan -st -f 5", true, expectedFew); + // also prove that BinaryFormatter behaves same as the default + exec("scan -st -fm org.apache.accumulo.core.util.format.BinaryFormatter", true, expected); + exec("scan -st -f 5 -fm org.apache.accumulo.core.util.format.BinaryFormatter", true, expectedFew); + exec("setauths -c", true); + exec("deletetable test -f", true, "Table: [test] has been deleted"); + } + + @Test + public void scanDateStringFormatterTest() throws IOException { + Shell.log.debug("Starting scan dateStringFormatter test --------------------------"); + exec("createtable t", true); + exec("insert r f q v -ts 0", true); + @SuppressWarnings("deprecation") + DateFormat dateFormat = new SimpleDateFormat(org.apache.accumulo.core.util.format.DateStringFormatter.DATE_FORMAT); + String expected = String.format("r f:q [] %s v", dateFormat.format(new Date(0))); + // historically, showing few did not pertain to ColVis or Timestamp + String expectedFew = expected; + String expectedNoTimestamp = String.format("r f:q [] v"); + exec("scan -fm org.apache.accumulo.core.util.format.DateStringFormatter -st", true, expected); + exec("scan -fm org.apache.accumulo.core.util.format.DateStringFormatter -st -f 1000", true, expected); + exec("scan -fm org.apache.accumulo.core.util.format.DateStringFormatter -st -f 5", true, expectedFew); + exec("scan -fm org.apache.accumulo.core.util.format.DateStringFormatter", true, expectedNoTimestamp); + exec("deletetable t -f", true, "Table: [t] has been deleted"); + } + + @Test + public void grepTest() throws IOException { + Shell.log.debug("Starting grep test --------------------------"); + exec("grep", false, "java.lang.IllegalStateException: Not in a table context"); + exec("createtable t", true); + exec("setauths -s vis", true); + exec("insert r f q v -ts 0 -l vis", true); + + String expected = "r f:q [vis] v"; + String expectedTimestamp = "r f:q [vis] 0 v"; + exec("grep", false, "No terms specified"); + exec("grep non_matching_string", true, ""); + // historically, showing few did not pertain to ColVis or Timestamp + exec("grep r", true, expected); + exec("grep r -f 1", true, expected); + exec("grep r -st", true, expectedTimestamp); + exec("grep r -st -f 1", true, expectedTimestamp); + exec("setauths -c", true); + exec("deletetable t -f", true, "Table: [t] has been deleted"); + } + + @Test + public void commentTest() throws IOException { + Shell.log.debug("Starting comment test --------------------------"); + exec("#", true, "Unknown command", false); + exec("# foo", true, "Unknown command", false); + exec("- foo", true, "Unknown command", true); + } + + @Test + public void execFileTest() throws IOException { + Shell.log.debug("Starting exec file test --------------------------"); + shell.config("--config-file", config.toString(), "--fake", "-u", "test", "-p", "secret", "-f", "src/test/resources/shelltest.txt"); + assertEquals(0, shell.start()); + assertGoodExit("Unknown command", false); + } + + @Test + public void setIterTest() throws IOException { + Shell.log.debug("Starting setiter test --------------------------"); + exec("createtable t", true); + + String cmdJustClass = "setiter -class VersioningIterator -p 1"; + exec(cmdJustClass, false, "java.lang.IllegalArgumentException", false); + exec(cmdJustClass, false, "fully qualified package name", true); + + String cmdFullPackage = "setiter -class o.a.a.foo -p 1"; + exec(cmdFullPackage, false, "java.lang.IllegalArgumentException", false); + exec(cmdFullPackage, false, "class not found", true); + + String cmdNoOption = "setiter -class java.lang.String -p 1"; + exec(cmdNoOption, false, "loaded successfully but does not implement SortedKeyValueIterator", true); + + input.set("\n\n"); + exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name foo", true); + + input.set("bar\nname value\n"); + exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 31", true); + + // TODO can't verify this as config -t fails, functionality verified in ShellServerIT + + exec("deletetable t -f", true, "Table: [t] has been deleted"); + } +}
