ACCUMULO-2947 Allow configuration of number of cells per mutation in mem stress test
A new switch to the memory stress test Write utility, --max-cells-per-mutation, controls the maximum number of cells to be written in any mutation. If the maximum is hit before the configured row width, the current mutation is used and the next mutation picks up where it left off, in the same row. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/75c053d8 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/75c053d8 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/75c053d8 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 75c053d8f312020ea0e9ffcc66e26c9322ce380d Parents: d4d455f Author: Bill Havanki <bhava...@cloudera.com> Authored: Thu Jun 26 17:21:05 2014 -0400 Committer: Bill Havanki <bhava...@cloudera.com> Committed: Tue Jul 1 15:20:56 2014 -0400 ---------------------------------------------------------------------- .../test/stress/random/RandomMutations.java | 21 ++++++++++++++++---- .../accumulo/test/stress/random/Write.java | 5 ++++- .../test/stress/random/WriteOptions.java | 3 +++ test/system/stress/stress-env.sh.example | 4 ++++ test/system/stress/writer.sh | 2 +- 5 files changed, 29 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java index c4504b2..679b983 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java @@ -21,24 +21,37 @@ import org.apache.accumulo.core.data.Mutation; public class RandomMutations extends Stream<Mutation> { private final RandomByteArrays rows, column_families, column_qualifiers, values; private final RandomWithinRange row_widths; + private final int max_cells_per_mutation; + private byte[] current_row; + private int cells_remaining_in_row; public RandomMutations(RandomByteArrays rows, RandomByteArrays column_families, - RandomByteArrays column_qualifiers, RandomByteArrays values, RandomWithinRange row_widths) { + RandomByteArrays column_qualifiers, RandomByteArrays values, RandomWithinRange row_widths, + int max_cells_per_mutation) { this.rows = rows; this.column_families = column_families; this.column_qualifiers = column_qualifiers; this.values = values; this.row_widths = row_widths; + this.max_cells_per_mutation = (max_cells_per_mutation > 0 ? max_cells_per_mutation : Integer.MAX_VALUE); + + current_row = null; + cells_remaining_in_row = 0; } // TODO should we care about timestamps? @Override public Mutation next() { - Mutation m = new Mutation(rows.next()); - final int cells = row_widths.next(); - for(int i = 0; i < cells; ++i) { + if (cells_remaining_in_row == 0) { + current_row = rows.next(); + cells_remaining_in_row = row_widths.next(); + } + Mutation m = new Mutation(current_row); + final int cells = Math.min(cells_remaining_in_row, max_cells_per_mutation); + for(int i = 1; i <= cells; i++) { m.put(column_families.next(), column_qualifiers.next(), values.next()); } + cells_remaining_in_row -= cells; return m; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java index 9c29871..bb679ad 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java @@ -83,7 +83,10 @@ public class Write { new RandomWithinRange( opts.row_width_seed, opts.rowWidthMin(), - opts.rowWidthMax()))); + opts.rowWidthMax()), + // max cells per mutation + opts.max_cells_per_mutation) + ); while(true) { dw.next(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java index c213528..3e6e647 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java @@ -72,6 +72,9 @@ class WriteOptions extends ClientOnDefaultTable { @Parameter(names = "--row-width-seed", description = "seed for generating the number of cells within a row (a row's \"width\")") int row_width_seed = 444; + @Parameter(names = "--max-cells-per-mutation", description = "maximum number of cells per mutation; non-positive value implies no limit") + int max_cells_per_mutation = -1; + @Parameter(names = "--write-delay", description = "milliseconds to wait between writes") long write_delay = 0L; http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/system/stress/stress-env.sh.example ---------------------------------------------------------------------- diff --git a/test/system/stress/stress-env.sh.example b/test/system/stress/stress-env.sh.example index 1360c67..6efb24f 100644 --- a/test/system/stress/stress-env.sh.example +++ b/test/system/stress/stress-env.sh.example @@ -52,5 +52,9 @@ CQ_SEED='--cq-seed 3' VALUE_SEED='--value-seed 4' ROW_WIDTH_SEED='--row-width-seed 5' +# This is the maximum number of cells to include in each mutation written out. +# A non-positive value implies no limit. +MAX_CELLS_PER_MUTATION='--max-cells-per-mutation -1' + # This is the delay in milliseconds between writes. Use <= 0 for no delay. WRITE_DELAY='--write-delay 0' http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/system/stress/writer.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/writer.sh b/test/system/stress/writer.sh index 7d9b283..ef26fed 100755 --- a/test/system/stress/writer.sh +++ b/test/system/stress/writer.sh @@ -37,6 +37,6 @@ host=$(hostname) # TBD - --clear-table option ${ACCUMULO_HOME}/bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE \ - $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED $WRITE_DELAY \ + $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED $MAX_CELLS_PER_MUTATION $WRITE_DELAY \ > $LOG_DIR/${ts}_${host}_writer.out \ 2> $LOG_DIR/${ts}_${host}_writer.err
