ACCUMULO-3871 fixed up a few more tests
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b7a529b7 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b7a529b7 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b7a529b7 Branch: refs/heads/master Commit: b7a529b75e335c577c53c77dfd1aee38a1cadcf6 Parents: b5004a1 Author: Eric C. Newton <eric.new...@gmail.com> Authored: Mon Jun 15 17:52:38 2015 -0400 Committer: Eric C. Newton <eric.new...@gmail.com> Committed: Mon Jun 15 17:52:38 2015 -0400 ---------------------------------------------------------------------- .../client/impl/ReplicationOperationsImpl.java | 1 - mrit/.gitignore | 1 + .../accumulo/test/AccumuloOutputFormatIT.java | 73 +++++++------------- .../accumulo/test/UserCompactionStrategyIT.java | 4 +- .../apache/accumulo/test/functional/BulkIT.java | 12 +--- .../accumulo/test/functional/ClassLoaderIT.java | 3 +- .../test/functional/GarbageCollectorIT.java | 19 +++-- .../apache/accumulo/test/functional/SslIT.java | 5 +- .../test/performance/RollWALPerformanceIT.java | 9 +-- test/src/main/resources/log4j.properties | 2 +- 10 files changed, 50 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java index 925877d..009479e 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java @@ -175,7 +175,6 @@ public class ReplicationOperationsImpl implements ReplicationOperations { } finally { metaBs.close(); } - return wals; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/mrit/.gitignore ---------------------------------------------------------------------- diff --git a/mrit/.gitignore b/mrit/.gitignore index 56204d2..db98860 100644 --- a/mrit/.gitignore +++ b/mrit/.gitignore @@ -15,6 +15,7 @@ # Maven ignores /target/ +/dependency-reduced-pom.xml # IDE ignores /.settings/ http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java b/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java index a2f522e..6b5e6eb 100644 --- a/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java +++ b/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java @@ -16,75 +16,46 @@ */ package org.apache.accumulo.test; -import static com.google.common.base.Charsets.UTF_8; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import java.io.File; import java.io.IOException; -import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.mapred.AccumuloOutputFormat; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.TablePermission; -import org.apache.accumulo.minicluster.MiniAccumuloCluster; -import org.apache.accumulo.minicluster.MiniAccumuloConfig; +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; +import org.apache.accumulo.test.functional.ConfigurableMacBase; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordWriter; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; - -import com.google.common.collect.Maps; /** - * Prevent regression of ACCUMULO-3709. Exists as a mini test because mock instance doesn't produce this error when dynamically changing the table permissions. + * Prevent regression of ACCUMULO-3709. */ -public class AccumuloOutputFormatIT { +public class AccumuloOutputFormatIT extends ConfigurableMacBase { private static final String TABLE = "abc"; - private MiniAccumuloCluster accumulo; - private String secret = "secret"; - - @Rule - public TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Before - public void setUp() throws Exception { - folder.create(); - MiniAccumuloConfig config = new MiniAccumuloConfig(folder.getRoot(), secret); - Map<String,String> configMap = Maps.newHashMap(); - configMap.put(Property.TSERV_SESSION_MAXIDLE.toString(), "1"); - config.setSiteConfig(configMap); - config.setNumTservers(1); - accumulo = new MiniAccumuloCluster(config); - accumulo.start(); - } - - @After - public void tearDown() throws Exception { - accumulo.stop(); - folder.delete(); + @Override + protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { + cfg.setProperty(Property.TSERV_SESSION_MAXIDLE, "1"); + cfg.setNumTservers(1); } @Test public void testMapred() throws Exception { - ClientConfiguration clientConfig = accumulo.getClientConfig(); - ZooKeeperInstance instance = new ZooKeeperInstance(clientConfig); - Connector connector = instance.getConnector("root", new PasswordToken(secret)); + Connector connector = getConnector(); // create a table and put some data in it connector.tableOperations().create(TABLE); @@ -98,8 +69,8 @@ public class AccumuloOutputFormatIT { batchConfig.setMaxMemory(Long.MAX_VALUE); AccumuloOutputFormat outputFormat = new AccumuloOutputFormat(); AccumuloOutputFormat.setBatchWriterOptions(job, batchConfig); - AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); - AccumuloOutputFormat.setConnectorInfo(job, "root", new PasswordToken(secret)); + AccumuloOutputFormat.setZooKeeperInstance(job, cluster.getClientConfig()); + AccumuloOutputFormat.setConnectorInfo(job, "root", new PasswordToken(ROOT_PASSWORD)); RecordWriter<Text,Mutation> writer = outputFormat.getRecordWriter(null, job, "Test", null); try { @@ -107,8 +78,8 @@ public class AccumuloOutputFormatIT { Mutation m = new Mutation(new Text(String.format("%08d", i))); for (int j = 0; j < 3; j++) { m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8))); - writer.write(new Text(TABLE), m); } + writer.write(new Text(TABLE), m); } } catch (Exception e) { @@ -118,8 +89,12 @@ public class AccumuloOutputFormatIT { connector.securityOperations().revokeTablePermission("root", TABLE, TablePermission.WRITE); - exception.expect(IOException.class); - exception.expectMessage("PERMISSION_DENIED"); - writer.close(null); + try { + writer.close(null); + fail("Did not throw exception"); + } catch (IOException ex) { + log.info(ex.getMessage(), ex); + assertTrue(ex.getCause() instanceof MutationsRejectedException); + } } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java b/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java index 9c319d2..588a83b 100644 --- a/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java @@ -128,12 +128,12 @@ public class UserCompactionStrategyIT extends AccumuloClusterHarness { // Can't assume that a test-resource will be on the server's classpath Assume.assumeTrue(ClusterType.MINI == getClusterType()); - // test pertable classpath + user specified compaction strat + // test per-table classpath + user specified compaction strategy final Connector c = getConnector(); final String tableName = getUniqueNames(1)[0]; File target = new File(System.getProperty("user.dir"), "target"); - Assert.assertTrue(target.mkdirs()); + target.mkdirs(); Assert.assertTrue(target.exists() && target.isDirectory()); File destFile = installJar(target, "/TestCompactionStrat.jar"); c.tableOperations().create(tableName); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java index f60724e..04570a4 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java @@ -26,10 +26,8 @@ import org.apache.accumulo.test.TestIngest.Opts; import org.apache.accumulo.test.VerifyIngest; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -67,7 +65,6 @@ public class BulkIT extends AccumuloClusterHarness { static void runTest(Connector c, FileSystem fs, Path basePath, String principal, String tableName, String filePrefix, String dirSuffix) throws Exception { c.tableOperations().create(tableName); - CachedConfiguration.setInstance(fs.getConf()); Path base = new Path(basePath, "testBulkFail_" + dirSuffix); fs.delete(base, true); @@ -84,24 +81,21 @@ public class BulkIT extends AccumuloClusterHarness { opts.instance = c.getInstance().getInstanceName(); opts.cols = 1; opts.setTableName(tableName); - opts.conf = CachedConfiguration.getInstance(); + opts.conf = new Configuration(false); opts.fs = fs; String fileFormat = filePrefix + "rf%02d"; for (int i = 0; i < COUNT; i++) { opts.outputFile = new Path(files, String.format(fileFormat, i)).toString(); opts.startRow = N * i; - TestIngest.ingest(c, opts, BWOPTS); + TestIngest.ingest(c, fs, opts, BWOPTS); } opts.outputFile = base + String.format(fileFormat, N); opts.startRow = N; opts.rows = 1; // create an rfile with one entry, there was a bug with this: - TestIngest.ingest(c, opts, BWOPTS); + TestIngest.ingest(c, fs, opts, BWOPTS); // Make sure the server can modify the files - FsShell fsShell = new FsShell(fs.getConf()); - Assert.assertEquals("Failed to chmod " + base.toString(), 0, fsShell.run(new String[] {"-chmod", "-R", "777", base.toString()})); - c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(), false); VerifyIngest.Opts vopts = new VerifyIngest.Opts(); vopts.setTableName(tableName); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java index f1fb91c..ebb0950 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java @@ -38,7 +38,6 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.harness.AccumuloClusterHarness; import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl; @@ -92,7 +91,7 @@ public class ClassLoaderIT extends AccumuloClusterHarness { bw.addMutation(m); bw.close(); scanCheck(c, tableName, "Test"); - FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); + FileSystem fs = getCluster().getFileSystem(); Path jarPath = new Path(rootPath + "/lib/ext/Test.jar"); copyStreamToFileSystem(fs, this.getClass().getResourceAsStream("/TestCombinerX.jar"), jarPath); UtilWaitThread.sleep(1000); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java index a73f239..6de8e93 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java @@ -16,6 +16,7 @@ */ package org.apache.accumulo.test.functional; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -43,7 +44,6 @@ import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.TablePermission; -import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.ServerServices; import org.apache.accumulo.core.util.ServerServices.Service; import org.apache.accumulo.core.util.UtilWaitThread; @@ -59,7 +59,6 @@ import org.apache.accumulo.server.zookeeper.ZooReaderWriter; import org.apache.accumulo.test.TestIngest; import org.apache.accumulo.test.VerifyIngest; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.io.Text; @@ -119,7 +118,7 @@ public class GarbageCollectorIT extends ConfigurableMacBase { vopts.cols = opts.cols = 1; opts.setPrincipal("root"); vopts.setPrincipal("root"); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, cluster.getFileSystem(), opts, new BatchWriterOpts()); c.tableOperations().compact("test_ingest", null, null, true, true); int before = countFiles(); while (true) { @@ -148,7 +147,16 @@ public class GarbageCollectorIT extends ConfigurableMacBase { cluster.getConfig().setDefaultMemory(10, MemoryUnit.MEGABYTE); Process gc = cluster.exec(SimpleGarbageCollector.class); UtilWaitThread.sleep(20 * 1000); - String output = FunctionalTestUtils.readAll(cluster, SimpleGarbageCollector.class, gc); + String output = ""; + while (!output.contains("delete candidates has exceeded")) { + byte buffer[] = new byte[10 * 1024]; + try { + int n = gc.getInputStream().read(buffer); + output = new String(buffer, 0, n, UTF_8); + } catch (IOException ex) { + break; + } + } gc.destroy(); assertTrue(output.contains("delete candidates has exceeded")); } @@ -279,9 +287,8 @@ public class GarbageCollectorIT extends ConfigurableMacBase { } private int countFiles() throws Exception { - FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); Path path = new Path(cluster.getConfig().getDir() + "/accumulo/tables/1/*/*.rf"); - return Iterators.size(Arrays.asList(fs.globStatus(path)).iterator()); + return Iterators.size(Arrays.asList(cluster.getFileSystem().globStatus(path)).iterator()); } public static void addEntries(Connector conn, BatchWriterOpts bwOpts) throws Exception { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java index 13248d0..b81b409 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java @@ -20,7 +20,6 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Test; @@ -60,8 +59,8 @@ public class SslIT extends ConfigurableMacBase { @Test public void bulk() throws Exception { - BulkIT.runTest(getConnector(), FileSystem.getLocal(new Configuration(false)), new Path(getCluster().getConfig().getDir().getAbsolutePath(), "tmp"), "root", - getUniqueNames(1)[0], this.getClass().getName(), testName.getMethodName()); + BulkIT.runTest(getConnector(), cluster.getFileSystem(), new Path(getCluster().getConfig().getDir().getAbsolutePath(), "tmp"), "root", getUniqueNames(1)[0], + this.getClass().getName(), testName.getMethodName()); } @Test http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java index a0d355e..bd42989 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java @@ -28,7 +28,6 @@ import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; -import org.apache.accumulo.minicluster.impl.ProcessReference; import org.apache.accumulo.test.continuous.ContinuousIngest; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.hadoop.conf.Configuration; @@ -40,7 +39,7 @@ public class RollWALPerformanceIT extends ConfigurableMacBase { @Override protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { cfg.setProperty(Property.TSERV_WAL_REPLICATION, "1"); - cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "10M"); + cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M"); cfg.setProperty(Property.TABLE_MINC_LOGS_MAX, "100"); cfg.setProperty(Property.GC_FILE_ARCHIVE, "false"); cfg.setProperty(Property.GC_CYCLE_START, "1s"); @@ -72,7 +71,7 @@ public class RollWALPerformanceIT extends ConfigurableMacBase { log.info("Starting ingest"); final long start = System.currentTimeMillis(); final String args[] = {"-i", inst.getInstanceName(), "-z", inst.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--batchThreads", "2", "--table", - tableName, "--num", Long.toString(1000 * 1000), // 1M 100 byte entries + tableName, "--num", Long.toString(50 * 1000), // 50K 100 byte entries }; ContinuousIngest.main(args); @@ -100,9 +99,7 @@ public class RollWALPerformanceIT extends ConfigurableMacBase { c.instanceOperations().setProperty(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1G"); c.tableOperations().flush(MetadataTable.NAME, null, null, true); c.tableOperations().flush(RootTable.NAME, null, null, true); - for (ProcessReference tserver : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) { - getCluster().killProcess(ServerType.TABLET_SERVER, tserver); - } + getCluster().getClusterControl().stop(ServerType.TABLET_SERVER); getCluster().start(); long avg2 = getAverage(); log.info(String.format("Average run time with small WAL %,d with large WAL %,d", avg1, avg2)); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/test/src/main/resources/log4j.properties b/test/src/main/resources/log4j.properties index 26ea762..c603b39 100644 --- a/test/src/main/resources/log4j.properties +++ b/test/src/main/resources/log4j.properties @@ -43,7 +43,7 @@ log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN log4j.logger.org.apache.hadoop.hdfs.server.datanode.DataNode.clienttrace=WARN log4j.logger.BlockStateChange=WARN log4j.logger.org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator=INFO -log4j.logger.org.apache.hadoop.security=DEBUG +log4j.logger.org.apache.hadoop.security=INFO log4j.logger.org.apache.hadoop.minikdc=DEBUG log4j.logger.org.apache.directory=INFO log4j.logger.org.apache.directory.api.ldap=WARN
