ACCUMULO-2849 Add scanner batch size and write delay configurability to mem stress test
This commit adds one new option each to the Scan and Write components of the memory stress test. The --scan-batch-size option for Scan configures the batch size of the scanner it uses, while the --write-delay option for Write allows for a delay between each mutation added to its batch writer. The reader.sh and writer.sh scripts are updated for easy configuration of these new options. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f7fe2a84 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f7fe2a84 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f7fe2a84 Branch: refs/heads/1.6.1-SNAPSHOT Commit: f7fe2a8475bf53da6f6f6e8374a39af0d4d9acbb Parents: 886cd19 Author: Bill Havanki <bhava...@cloudera.com> Authored: Tue Jun 24 21:17:57 2014 -0400 Committer: Bill Havanki <bhava...@cloudera.com> Committed: Tue Jun 24 21:17:57 2014 -0400 ---------------------------------------------------------------------- .../java/org/apache/accumulo/test/stress/random/Scan.java | 3 +++ .../org/apache/accumulo/test/stress/random/ScanOpts.java | 5 ++++- .../org/apache/accumulo/test/stress/random/Write.java | 10 +++++++++- .../apache/accumulo/test/stress/random/WriteOptions.java | 5 ++++- test/system/stress/reader.sh | 6 +++++- test/system/stress/writer.sh | 6 +++++- 6 files changed, 30 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java index 00a1cec..9408770 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java @@ -54,6 +54,9 @@ public class Scan { Range range = pickRange(connector.tableOperations(), opts.getTableName(), tablet_index_generator); scanner.setRange(range); + if (opts.batch_size > 0) { + scanner.setBatchSize(opts.batch_size); + } try { consume(scanner); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java b/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java index 22024d1..86e7920 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java @@ -33,7 +33,10 @@ class ScanOpts extends ClientOnDefaultTable { @Parameter(names = "--scan-seed", description = "seed for randomly choosing tablets to scan") int scan_seed = 1337; - + + @Parameter(names = "--scan-batch-size", description="scanner batch size") + int batch_size = -1; + public ScanOpts() { this(WriteOptions.DEFAULT_TABLE); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java index 3df9808..9c29871 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java @@ -47,7 +47,12 @@ public class Write { System.err.println("Couldn't create table ourselves, but that's ok. Continuing."); } } - + + long writeDelay = opts.write_delay; + if (writeDelay < 0) { + writeDelay = 0; + } + DataWriter dw = new DataWriter(c.createBatchWriter(opts.getTableName(), batch_writer_opts.getBatchWriterConfig()), new RandomMutations( //rows @@ -82,6 +87,9 @@ public class Write { while(true) { dw.next(); + if (writeDelay > 0) { + Thread.sleep(writeDelay); + } } } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java index 85ff25b..c213528 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java @@ -71,7 +71,10 @@ class WriteOptions extends ClientOnDefaultTable { @Parameter(names = "--row-width-seed", description = "seed for generating the number of cells within a row (a row's \"width\")") int row_width_seed = 444; - + + @Parameter(names = "--write-delay", description = "milliseconds to wait between writes") + long write_delay = 0L; + public WriteOptions(String table) { super(table); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/system/stress/reader.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/reader.sh b/test/system/stress/reader.sh index 8f10fa2..0b796f0 100755 --- a/test/system/stress/reader.sh +++ b/test/system/stress/reader.sh @@ -31,4 +31,8 @@ CONTINUOUS_SCAN='--continuous' # Controls whether or not the scan will be using an isolated scanner. Add this to the execution #SCAN_ISOLATION='--isolate' -../../../bin/accumulo org.apache.accumulo.test.stress.random.Scan $INSTANCE $USERPASS $SCAN_SEED $CONTINUOUS_SCAN \ No newline at end of file +# Sets the batch size for the scanner, use a lower number for large rows / cells +#SCAN_BATCH_SIZE='--scan-batch-size -1' + +../../../bin/accumulo org.apache.accumulo.test.stress.random.Scan $INSTANCE $USERPASS $SCAN_SEED $CONTINUOUS_SCAN \ + $SCAN_BATCH_SIZE http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/system/stress/writer.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/writer.sh b/test/system/stress/writer.sh index 8265a68..db55155 100755 --- a/test/system/stress/writer.sh +++ b/test/system/stress/writer.sh @@ -33,8 +33,12 @@ CQ_SEED='--cq-seed 3' VALUE_SEED='--value-seed 4' ROW_WIDTH_SEED='--row-width-seed 5' +# This is the delay in milliseconds between writes. Use <= 0 for no delay. +WRITE_DELAY='--write-delay 0' + # Let's reset the table, for good measure ../../../bin/accumulo shell $USERPASS -e 'deletetable -f stress_test' ../../../bin/accumulo shell $USERPASS -e 'createtable stress_test' -../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED +../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE \ + $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED $WRITE_DELAY
