ACCUMULO-2789 Added parameter to control the number of cells within a row When running the stress test, it is now possible to configure the writer to send mutations for rows that contain a random amount of column updates. This is controlled via the --min-row-width, --max-row-width and --row-width-seed parameters.
I did some internal shuffling to make randomly generating numbers within a range more convenient. See the new RandomWithinRange class for details. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f0dcf1e5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f0dcf1e5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f0dcf1e5 Branch: refs/heads/1.5.2-SNAPSHOT Commit: f0dcf1e5570c12098e716ec58ea2f4bc3321053b Parents: 6970d73 Author: Bill Slacum <ujustgotbi...@apache.org> Authored: Fri May 23 19:11:39 2014 -0400 Committer: Bill Slacum <ujustgotbi...@apache.org> Committed: Fri May 23 19:11:39 2014 -0400 ---------------------------------------------------------------------- .../random/RandomByteArrayMakerFunction.java | 60 -------------------- .../test/stress/random/RandomByteArrays.java | 14 ++--- .../test/stress/random/RandomMutations.java | 11 +++- .../test/stress/random/RandomWithinRange.java | 43 ++++++++++++++ .../accumulo/test/stress/random/Write.java | 33 +++++++---- .../test/stress/random/WriteOptions.java | 17 ++++++ test/system/stress/writer.sh | 4 +- 7 files changed, 97 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java deleted file mode 100644 index bfe0be8..0000000 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.stress.random; - -import java.util.Random; - -import com.google.common.base.Preconditions; - -/** - * This class is one RBAMF. Don't mess with it, or it'll mess with you. - */ -public class RandomByteArrayMakerFunction { - private final Random random; - - public RandomByteArrayMakerFunction() { - this(0); - } - - public RandomByteArrayMakerFunction(int seed) { - random = new Random(seed); - } - - public byte[] make(int size) { - byte[] b = new byte[size]; - random.nextBytes(b); - return b; - } - - public byte[] makeWithRandomSize(int max) { - return makeWithRandomSize(1, random.nextInt(max)); - } - - public byte[] makeWithRandomSize(int min, int max) { - Preconditions.checkArgument(min > 0, "Min must be positive."); - Preconditions.checkArgument(max >= min, "Max must be greater than or equal to min."); - if (min == max) { - return make(min); - } else { - final int spread = max - min; - final int random_value = random.nextInt(spread); - // we pick a random number that's between 0 and (max - min), then add - // min as an offset to get a random number that's [min, max) - return make(random_value + min); - } - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java index 62f7cb2..0b6b36a 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java @@ -17,20 +17,16 @@ package org.apache.accumulo.test.stress.random; /** - * An iterator that will create random byte arrays as it is looped over. + * A stream that will create random byte arrays as it is looped over. */ public class RandomByteArrays extends Stream<byte[]> { - private final RandomByteArrayMakerFunction rbamf; - private final int min_array_size, max_array_size; + private final RandomWithinRange random_arrays; - public RandomByteArrays(RandomByteArrayMakerFunction rbamf, int min_array_size, - int max_array_size) { - this.rbamf = rbamf; - this.min_array_size = min_array_size; - this.max_array_size = max_array_size; + public RandomByteArrays(RandomWithinRange random_arrays) { + this.random_arrays = random_arrays; } public byte[] next() { - return rbamf.makeWithRandomSize(min_array_size, max_array_size); + return random_arrays.next_bytes(); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java index a22222b..c4504b2 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java @@ -20,20 +20,25 @@ import org.apache.accumulo.core.data.Mutation; public class RandomMutations extends Stream<Mutation> { private final RandomByteArrays rows, column_families, column_qualifiers, values; + private final RandomWithinRange row_widths; public RandomMutations(RandomByteArrays rows, RandomByteArrays column_families, - RandomByteArrays column_qualifiers, RandomByteArrays values) { + RandomByteArrays column_qualifiers, RandomByteArrays values, RandomWithinRange row_widths) { this.rows = rows; this.column_families = column_families; this.column_qualifiers = column_qualifiers; this.values = values; + this.row_widths = row_widths; } - + // TODO should we care about timestamps? @Override public Mutation next() { Mutation m = new Mutation(rows.next()); - m.put(column_families.next(), column_qualifiers.next(), values.next()); + final int cells = row_widths.next(); + for(int i = 0; i < cells; ++i) { + m.put(column_families.next(), column_qualifiers.next(), values.next()); + } return m; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java new file mode 100644 index 0000000..c66cf38 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java @@ -0,0 +1,43 @@ +package org.apache.accumulo.test.stress.random; + +import java.util.Random; + +import com.google.common.base.Preconditions; + +/** + * Class that returns positive integers between some minimum + * and maximum. + * + */ +public class RandomWithinRange { + private final Random random; + private final int min, max; + + public RandomWithinRange(int seed, int min, int max) { + this(new Random(seed), min, max); + } + + public RandomWithinRange(Random random, int min, int max) { + Preconditions.checkArgument(min > 0, "Min must be positive."); + Preconditions.checkArgument(max >= min, "Max must be greater than or equal to min."); + this.random = random; + this.min = min; + this.max = max; + } + + public int next() { + if (min == max) { + return min; + } else { + // we pick a random number that's between 0 and (max - min), then add + // min as an offset to get a random number that's [min, max) + return random.nextInt(max - min) + min; + } + } + + public byte[] next_bytes() { + byte[] b = new byte[next()]; + random.nextBytes(b); + return b; + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java index 2c817c3..3df9808 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java @@ -52,24 +52,33 @@ public class Write { new RandomMutations( //rows new RandomByteArrays( - new RandomByteArrayMakerFunction(opts.row_seed), - opts.rowMin(), - opts.rowMax()), + new RandomWithinRange( + opts.row_seed, + opts.rowMin(), + opts.rowMax())), //cfs new RandomByteArrays( - new RandomByteArrayMakerFunction(opts.cf_seed), - opts.cfMin(), - opts.cfMax()), + new RandomWithinRange( + opts.cf_seed, + opts.cfMin(), + opts.cfMax())), //cqs new RandomByteArrays( - new RandomByteArrayMakerFunction(opts.cq_seed), - opts.cqMin(), - opts.cqMax()), + new RandomWithinRange( + opts.cq_seed, + opts.cqMin(), + opts.cqMax())), //vals new RandomByteArrays( - new RandomByteArrayMakerFunction(opts.value_seed), - opts.valueMin(), - opts.valueMax()))); + new RandomWithinRange( + opts.value_seed, + opts.valueMin(), + opts.valueMax())), + //number of cells per row + new RandomWithinRange( + opts.row_width_seed, + opts.rowWidthMin(), + opts.rowWidthMax()))); while(true) { dw.next(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java index b850e85..85ff25b 100644 --- a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java +++ b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java @@ -48,6 +48,12 @@ class WriteOptions extends ClientOnDefaultTable { @Parameter(validateValueWith = IntArgValidator.class, names = "--max-value-size", description = "maximum value size") Integer value_max; + @Parameter(validateValueWith = IntArgValidator.class, names = "--min-row-width", description = "minimum row width") + Integer row_width_min; + + @Parameter(validateValueWith = IntArgValidator.class, names = "--max-row-width", description = "maximum row width") + Integer row_width_max; + @Parameter(names = "--clear-table", description = "clears the table before ingesting") boolean clear_table; @@ -63,6 +69,9 @@ class WriteOptions extends ClientOnDefaultTable { @Parameter(names = "--value-seed", description = "seed for generating values") int value_seed = 99; + @Parameter(names = "--row-width-seed", description = "seed for generating the number of cells within a row (a row's \"width\")") + int row_width_seed = 444; + public WriteOptions(String table) { super(table); } @@ -149,4 +158,12 @@ class WriteOptions extends ClientOnDefaultTable { public int valueMax() { return calculateMax(value_min, value_max); } + + public int rowWidthMin() { + return minOrDefault(row_width_min); + } + + public int rowWidthMax() { + return calculateMax(row_width_min, row_width_max); + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/system/stress/writer.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/writer.sh b/test/system/stress/writer.sh index 6054ab1..8265a68 100755 --- a/test/system/stress/writer.sh +++ b/test/system/stress/writer.sh @@ -24,15 +24,17 @@ ROW_RANGE='--min-row-size 128 --max-row-size 128' CF_RANGE='--min-cf-size 128 --max-cf-size 128' CQ_RANGE='--min-cq-size 128 --max-cq-size 128' VALUE_RANGE='--min-value-size 1024 --max-value-size 2048' +ROW_WIDTH='--min-row-width 1 --max-row-width 10' # These are the seeds for the random number generates used to generate each cell component. ROW_SEED='--row-seed 1' CF_SEED='--cf-seed 2' CQ_SEED='--cq-seed 3' VALUE_SEED='--value-seed 4' +ROW_WIDTH_SEED='--row-width-seed 5' # Let's reset the table, for good measure ../../../bin/accumulo shell $USERPASS -e 'deletetable -f stress_test' ../../../bin/accumulo shell $USERPASS -e 'createtable stress_test' -../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED +../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED
