This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 427a6fa Enhance GenerateData to produce deterministic time-series
(#5497)
427a6fa is described below
commit 427a6faa22a14776e5c7968571a60e7c5cbee038
Author: Alexander Pucher <apuc...@apache.org>
AuthorDate: Wed Jun 10 12:26:19 2020 -0700
Enhance GenerateData to produce deterministic time-series (#5497)
We enhance the pinot-tools GenerateData command to produce pseudo-random
(deterministic) time-series for templates and support user-specified table
names in bin/generator.sh.
---
docker/images/pinot/bin/generator.sh | 43 +++++---
.../data/generator/PatternSeasonalGenerator.java | 6 +-
.../data/generator/PatternSpikeGenerator.java | 42 ++++++--
.../generator/complexWebsite_generator.json | 108 ++++++++++-----------
.../src/main/resources/generator/generator.sh | 76 ---------------
5 files changed, 121 insertions(+), 154 deletions(-)
diff --git a/docker/images/pinot/bin/generator.sh
b/docker/images/pinot/bin/generator.sh
index dc84967..a79afa1 100755
--- a/docker/images/pinot/bin/generator.sh
+++ b/docker/images/pinot/bin/generator.sh
@@ -20,35 +20,42 @@
JAR_PATH="$(find /opt/pinot/lib/pinot-all-*-jar-with-dependencies.jar)"
ADMIN_PATH="/opt/pinot/bin/pinot-admin.sh"
-TEMPLATE_BASEDIR="/tmp/pinotGenerator/generator"
-TEMP_DIR="/tmp/pinotGenerator"
+TEMP_DIR=$(mktemp -d -t pinotGenerator-XXXXXXXX)
+TEMPLATE_BASEDIR="$TEMP_DIR/generator"
-if [ -z "$1" ]; then
+TEMPLATE_NAME="$1"
+if [ -z "$TEMPLATE_NAME" ]; then
echo "No template name specified. Aborting."
exit 1
fi
-TEMPLATE_NAME="$1"
+TABLE_NAME="$2"
+if [ -z "$TABLE_NAME" ]; then
+ echo "No table name specified. Defaulting to '$TEMPLATE_NAME'"
+ TABLE_NAME=$TEMPLATE_NAME
+fi
+
DATA_DIR="${TEMP_DIR:?}/${TEMPLATE_NAME}"
SEGMENT_DIR="${TEMP_DIR:?}/${TEMPLATE_NAME}Segment"
-echo "Preparing temp directory for ${TEMPLATE_NAME}"
-rm -rf "${DATA_DIR}"
-rm -rf "${SEGMENT_DIR}"
-mkdir -p "${TEMP_DIR}"
-
-echo "Extracting template files"
+echo "Extracting template files to '${TEMP_DIR}'"
/bin/sh -c "cd \"${TEMP_DIR}\" && jar -xf \"${JAR_PATH}\"
\"generator/${TEMPLATE_NAME}_schema.json\"
\"generator/${TEMPLATE_NAME}_config.json\"
\"generator/${TEMPLATE_NAME}_generator.json\""
+echo "Setting table name and schema name to $TABLE_NAME"
+sed -i -e "s/\"tableName\": \"$TEMPLATE_NAME\"/\"tableName\":
\"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json"
+sed -i -e "s/\"schemaName\": \"$TEMPLATE_NAME\"/\"schemaName\":
\"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json"
+sed -i -e "s/\"schemaName\": \"$TEMPLATE_NAME\"/\"schemaName\":
\"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json"
+
echo "Generating data for ${TEMPLATE_NAME} in ${DATA_DIR}"
${ADMIN_PATH} GenerateData \
--numFiles 1 -numRecords 354780 -format csv \
+-numFiles 1 -numRecords 631152 -format csv \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" \
-schemaAnnotationFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_generator.json" \
-outDir "$DATA_DIR"
if [ ! -d "${DATA_DIR}" ]; then
echo "Data generation failed. Aborting."
+ rm -rf "$TEMP_DIR"
exit 1
fi
@@ -58,21 +65,25 @@ ${ADMIN_PATH} CreateSegment \
-tableConfigFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json" \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" \
-dataDir "${DATA_DIR}" \
--outDir "${SEGMENT_DIR}" || exit 1
+-outDir "${SEGMENT_DIR}"
if [ ! -d "${SEGMENT_DIR}" ]; then
echo "Data generation failed. Aborting."
+ rm -rf "$TEMP_DIR"
exit 1
fi
-echo "Adding table ${TEMPLATE_NAME}"
+echo "Adding table ${TABLE_NAME} from template ${TEMPLATE_NAME}"
${ADMIN_PATH} AddTable -exec \
-tableConfigFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json" \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" || exit 1
echo "Uploading segment for ${TEMPLATE_NAME}"
${ADMIN_PATH} UploadSegment \
--tableName "${TEMPLATE_NAME}" \
--segmentDir "${SEGMENT_DIR}" || exit 1
+-tableName "${TABLE_NAME}" \
+-segmentDir "${SEGMENT_DIR}"
+
+echo "Deleting temp directory"
+rm -rf "$TEMP_DIR"
-echo "Succesfully applied template ${TEMPLATE_NAME}"
+echo "Succesfully created table ${TABLE_NAME} from template ${TEMPLATE_NAME}"
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSeasonalGenerator.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSeasonalGenerator.java
index 36aa600..38dcd81 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSeasonalGenerator.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSeasonalGenerator.java
@@ -20,6 +20,7 @@ package org.apache.pinot.tools.data.generator;
import org.apache.commons.configuration.PropertyConverter;
import org.apache.commons.math3.distribution.NormalDistribution;
+import org.apache.commons.math3.random.Well19937c;
import java.util.Arrays;
import java.util.List;
@@ -64,18 +65,19 @@ public class PatternSeasonalGenerator implements Generator {
PropertyConverter.toDouble(templateConfig.getOrDefault("wavelength", 0)),
PropertyConverter.
toDouble(templateConfig.getOrDefault("amplitude", 0)),
PropertyConverter.
toDouble(templateConfig.getOrDefault("offset", 0)),
+ PropertyConverter.
toInteger(templateConfig.getOrDefault("seed", 0)),
toDoubleArray(templateConfig.get("scalingFactors"), 1));
}
public PatternSeasonalGenerator(double mean, double sigma, double trend,
double wavelength, double amplitude,
- double offset, double[] scalingFactors) {
+ double offset, int seed, double[]
scalingFactors) {
this.trend = trend;
this.wavelength = wavelength;
this.amplitude = amplitude;
this.offset = offset;
this.scalingFactors = scalingFactors;
- this.generator = new NormalDistribution(mean, sigma);
+ this.generator = new NormalDistribution(new Well19937c(seed), mean,
sigma, 1.0E-9D);
}
@Override
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSpikeGenerator.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSpikeGenerator.java
index 570551a..5b23e09 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSpikeGenerator.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSpikeGenerator.java
@@ -19,7 +19,11 @@
package org.apache.pinot.tools.data.generator;
import org.apache.commons.configuration.PropertyConverter;
+import org.apache.commons.math3.distribution.AbstractRealDistribution;
+import org.apache.commons.math3.distribution.ExponentialDistribution;
import org.apache.commons.math3.distribution.LogNormalDistribution;
+import org.apache.commons.math3.distribution.UniformRealDistribution;
+import org.apache.commons.math3.random.Well19937c;
import java.util.Map;
@@ -46,34 +50,60 @@ public class PatternSpikeGenerator implements Generator {
private final double baseline;
private final double smoothing;
- private final LogNormalDistribution arrivalGenerator;
- private final LogNormalDistribution magnitudeGenerator;
+ private final AbstractRealDistribution arrivalGenerator;
+ private final AbstractRealDistribution magnitudeGenerator;
private long step = -1;
private long nextArrival;
private double lastValue;
+ enum DistributionType {
+ LOGNORMAL,
+ EXPONENTIAL,
+ UNIFORM,
+ FIXED
+ }
+
public PatternSpikeGenerator(Map<String, Object> templateConfig) {
this(PropertyConverter.toDouble(templateConfig.getOrDefault("baseline", 0)),
+
DistributionType.valueOf(templateConfig.getOrDefault("arrivalType",
"lognormal").toString().toUpperCase()),
PropertyConverter.toDouble(templateConfig.getOrDefault("arrivalMean", 2)),
PropertyConverter.toDouble(templateConfig.getOrDefault("arrivalSigma", 1)),
+
DistributionType.valueOf(templateConfig.getOrDefault("magnitudeType",
"lognormal").toString().toUpperCase()),
PropertyConverter.toDouble(templateConfig.getOrDefault("magnitudeMean", 2)),
PropertyConverter.toDouble(templateConfig.getOrDefault("magnitudeSigma", 1)),
-
PropertyConverter.toDouble(templateConfig.getOrDefault("smoothing", 0)));
+
PropertyConverter.toDouble(templateConfig.getOrDefault("smoothing", 0)),
+
PropertyConverter.toInteger(templateConfig.getOrDefault("seed", 0)));
}
- public PatternSpikeGenerator(double baseline, double arrivalMean, double
arrivalSigma, double magnitudeMean, double magnitudeSigma, double smoothing) {
+ public PatternSpikeGenerator(double baseline, DistributionType
arrivalType, double arrivalMean, double arrivalSigma,
+ DistributionType magnitudeType, double
magnitudeMean, double magnitudeSigma, double smoothing, int seed) {
this.baseline = baseline;
this.smoothing = smoothing;
- this.arrivalGenerator = new LogNormalDistribution(arrivalMean,
arrivalSigma);
- this.magnitudeGenerator = new LogNormalDistribution(magnitudeMean,
magnitudeSigma);
+ this.arrivalGenerator = makeDist(arrivalType, arrivalMean,
arrivalSigma, seed);
+ this.magnitudeGenerator = makeDist(magnitudeType, magnitudeMean,
magnitudeSigma, seed);
this.nextArrival = (long) arrivalGenerator.sample();
this.lastValue = baseline;
}
+ private static AbstractRealDistribution makeDist(DistributionType type,
double mean, double sigma, int seed) {
+ switch (type) {
+ case LOGNORMAL:
+ return new LogNormalDistribution(new Well19937c(seed), mean,
sigma, 1.0E-9D);
+ case EXPONENTIAL:
+ return new ExponentialDistribution(new Well19937c(seed), mean,
1.0E-9D);
+ case UNIFORM:
+ return new UniformRealDistribution(new Well19937c(seed), mean
- sigma, mean + sigma);
+ case FIXED:
+ return new UniformRealDistribution(new Well19937c(seed), mean,
mean + 1.0E-9D);
+ default:
+ throw new IllegalArgumentException(String.format("Unsupported
distribution type '%s", type));
+ }
+ }
+
@Override
public void init() {
// left blank
diff --git
a/pinot-tools/src/main/resources/generator/complexWebsite_generator.json
b/pinot-tools/src/main/resources/generator/complexWebsite_generator.json
index 41f08c9..7777d33 100644
--- a/pinot-tools/src/main/resources/generator/complexWebsite_generator.json
+++ b/pinot-tools/src/main/resources/generator/complexWebsite_generator.json
@@ -31,24 +31,24 @@
"type": "SEASONAL", "wavelength": 24, "scalingFactors": [ 0.4, 0.9,
1.0, 1.0, 1.0, 0.8, 0.4 ]
},
"generatorBins": [
- [ { "mean": 50, "sigma": 4.0, "amplitude": 40, "offset": 0.0 } ],
- [ { "mean": 25, "sigma": 2.5, "amplitude": 15, "offset": 0.4 } ],
- [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6 } ],
- [ { "mean": 90, "sigma": 6.0, "amplitude": 80, "offset": 0.0 }, {
"type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 5,
"magnitudeSigma": 1, "smoothing": 0.7 } ],
- [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4 } ],
- [ { "mean": 20, "sigma": 1.5, "amplitude": 15, "offset": 0.6 } ],
- [ { "mean": 10, "sigma": 0.5, "amplitude": 8, "offset": 0.0 } ],
- [ { "mean": 25, "sigma": 2.5, "amplitude": 15, "offset": 0.4 } ],
- [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6 } ],
- [ { "mean": 20, "sigma": 1.0, "amplitude": 18, "offset": 0.0 }, {
"type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 0.5,
"magnitudeSigma": 0.5, "smoothing": 0.5 } ],
- [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4 } ],
- [ { "mean": 20, "sigma": 1.5, "amplitude": 15, "offset": 0.6 } ],
- [ { "mean": 50, "sigma": 4.0, "amplitude": 40, "offset": 0.0 } ],
- [ { "mean": 10, "sigma": 1.0, "amplitude": 8, "offset": 0.4 } ],
- [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6 } ],
- [ { "mean": 90, "sigma": 6.0, "amplitude": 80, "offset": 0.0 }, {
"type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 4,
"magnitudeSigma": 1, "smoothing": 0.5 } ],
- [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4 } ],
- [ { "mean": 10, "sigma": 0.5, "amplitude": 9, "offset": 0.6 } ]
+ [ { "mean": 50, "sigma": 4.0, "amplitude": 40, "offset": 0.0, "seed":
0 } ],
+ [ { "mean": 25, "sigma": 2.5, "amplitude": 15, "offset": 0.4, "seed":
1 } ],
+ [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6, "seed":
2 } ],
+ [ { "mean": 90, "sigma": 6.0, "amplitude": 80, "offset": 0.0, "seed":
3 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120,
"arrivalSigma": 1, "magnitudeMean": 5, "magnitudeSigma": 1, "smoothing": 0.7,
"seed": 3 } ],
+ [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4, "seed":
4 } ],
+ [ { "mean": 20, "sigma": 1.5, "amplitude": 15, "offset": 0.6, "seed":
5 } ],
+ [ { "mean": 10, "sigma": 0.5, "amplitude": 8, "offset": 0.0, "seed":
6 } ],
+ [ { "mean": 25, "sigma": 2.5, "amplitude": 15, "offset": 0.4, "seed":
7 } ],
+ [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6, "seed":
8 } ],
+ [ { "mean": 20, "sigma": 1.0, "amplitude": 18, "offset": 0.0, "seed":
9 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120,
"arrivalSigma": 1, "magnitudeMean": 0.5, "magnitudeSigma": 0.5, "smoothing":
0.5, "seed": 3 } ],
+ [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4, "seed":
10 } ],
+ [ { "mean": 20, "sigma": 1.5, "amplitude": 15, "offset": 0.6, "seed":
11 } ],
+ [ { "mean": 50, "sigma": 4.0, "amplitude": 40, "offset": 0.0, "seed":
12 } ],
+ [ { "mean": 10, "sigma": 1.0, "amplitude": 8, "offset": 0.4, "seed":
13 } ],
+ [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6, "seed":
14 } ],
+ [ { "mean": 90, "sigma": 6.0, "amplitude": 80, "offset": 0.0, "seed":
15 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120,
"arrivalSigma": 1, "magnitudeMean": 4, "magnitudeSigma": 1, "smoothing": 0.5,
"seed": 3 } ],
+ [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4, "seed":
16 } ],
+ [ { "mean": 10, "sigma": 0.5, "amplitude": 9, "offset": 0.6, "seed":
17 } ]
]
}
},
@@ -60,24 +60,24 @@
"type": "SEASONAL", "wavelength": 24, "scalingFactors": [ 0.5, 0.9,
1.0, 1.0, 1.0, 0.9, 0.5 ]
},
"generatorBins": [
- [ { "mean": 12, "sigma": 1.0, "amplitude": 10, "offset": 0.0 } ],
- [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4 } ],
- [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6 } ],
- [ { "mean": 20, "sigma": 2.0, "amplitude": 18, "offset": 0.0 }, {
"type": "SPIKE", "arrivalMean": 4, "arrivalSigma": 1, "magnitudeMean": 2,
"magnitudeSigma": 1, "smoothing": 0.5 } ],
- [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4 } ],
- [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6 } ],
- [ { "mean": 3, "sigma": 0.5, "amplitude": 2, "offset": 0.0 } ],
- [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4 } ],
- [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6 } ],
- [ { "mean": 5, "sigma": 0.5, "amplitude": 4, "offset": 0.0 }, {
"type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 0.5,
"magnitudeSigma": 0.5, "smoothing": 0.3 } ],
- [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4 } ],
- [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6 } ],
- [ { "mean": 12, "sigma": 1.0, "amplitude": 10, "offset": 0.0 } ],
- [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4 } ],
- [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6 } ],
- [ { "mean": 20, "sigma": 2.0, "amplitude": 18, "offset": 0.0 }, {
"type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 3,
"magnitudeSigma": 1, "smoothing": 0.3 } ],
- [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4 } ],
- [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6 } ]
+ [ { "mean": 12, "sigma": 1.0, "amplitude": 10, "offset": 0.0, "seed":
0 } ],
+ [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4, "seed":
1 } ],
+ [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6, "seed":
2 } ],
+ [ { "mean": 20, "sigma": 2.0, "amplitude": 18, "offset": 0.0, "seed":
3 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120,
"arrivalSigma": 1, "magnitudeMean": 2, "magnitudeSigma": 1, "smoothing": 0.5,
"seed": 3 } ],
+ [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4, "seed":
4 } ],
+ [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6, "seed":
5 } ],
+ [ { "mean": 3, "sigma": 0.5, "amplitude": 2, "offset": 0.0, "seed":
6 } ],
+ [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4, "seed":
7 } ],
+ [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6, "seed":
8 } ],
+ [ { "mean": 5, "sigma": 0.5, "amplitude": 4, "offset": 0.0, "seed":
9 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120,
"arrivalSigma": 1, "magnitudeMean": 0.5, "magnitudeSigma": 0.5, "smoothing":
0.3, "seed": 3 } ],
+ [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4, "seed":
10 } ],
+ [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6, "seed":
11 } ],
+ [ { "mean": 12, "sigma": 1.0, "amplitude": 10, "offset": 0.0, "seed":
12 } ],
+ [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4, "seed":
13 } ],
+ [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6, "seed":
14 } ],
+ [ { "mean": 20, "sigma": 2.0, "amplitude": 18, "offset": 0.0, "seed":
15 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120,
"arrivalSigma": 1, "magnitudeMean": 3, "magnitudeSigma": 1, "smoothing": 0.3,
"seed": 3 } ],
+ [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4, "seed":
16 } ],
+ [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6, "seed":
17 } ]
]
}
},
@@ -89,24 +89,24 @@
"type": "SPIKE", "arrivalMean": 4, "arrivalSigma": 1, "magnitudeMean":
2, "magnitudeSigma": 1, "smoothing": 0.3
},
"generatorBins": [
- [ { "arrivalMean": 4, "magnitudeMean": 2.0 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 1.0 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.2 } ],
- [ { "arrivalMean": 4, "magnitudeMean": 2.5 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 0.8 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.1 } ],
- [ { "arrivalMean": 1, "magnitudeMean": 0.5 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 1.0 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.2 } ],
- [ { "arrivalMean": 1, "magnitudeMean": 0.5 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 0.8 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.1 } ],
- [ { "arrivalMean": 4, "magnitudeMean": 2.0 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 1.0 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.2 } ],
- [ { "arrivalMean": 4, "magnitudeMean": 2.5 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 0.8 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.1 } ]
+ [ { "arrivalMean": 4, "magnitudeMean": 2.0, "seed": 0 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 1.0, "seed": 1 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.2, "seed": 2 } ],
+ [ { "arrivalMean": 4, "magnitudeMean": 2.5, "seed": 3 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 0.8, "seed": 4 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.1, "seed": 5 } ],
+ [ { "arrivalMean": 1, "magnitudeMean": 0.5, "seed": 6 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 1.0, "seed": 7 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.2, "seed": 8 } ],
+ [ { "arrivalMean": 1, "magnitudeMean": 0.5, "seed": 9 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 0.8, "seed": 10 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.1, "seed": 11 } ],
+ [ { "arrivalMean": 4, "magnitudeMean": 2.0, "seed": 12 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 1.0, "seed": 13 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.2, "seed": 14 } ],
+ [ { "arrivalMean": 4, "magnitudeMean": 2.5, "seed": 15 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 0.8, "seed": 16 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.1, "seed": 17 } ]
]
}
}
diff --git a/pinot-tools/src/main/resources/generator/generator.sh
b/pinot-tools/src/main/resources/generator/generator.sh
deleted file mode 100755
index ac1e746..0000000
--- a/pinot-tools/src/main/resources/generator/generator.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-#ADMIN_PATH="/opt/pinot/bin/pinot-admin.sh"
-ADMIN_PATH="./pinot-tools/target/pinot-tools-pkg/bin/pinot-admin.sh"
-#PATTERN_BASEDIR="/opt/pinot/examples/docker/generators"
-PATTERN_BASEDIR="./pinot-tools/src/main/resources/generator"
-TEMP_DIR="/tmp/pinotGenerator"
-
-if [ -z "$1" ]; then
- echo "No PATTERN name specified. Aborting."
- exit 1
-fi
-
-PATTERN_NAME="$1"
-DATA_DIR="${TEMP_DIR:?}/${PATTERN_NAME}"
-SEGMENT_DIR="${TEMP_DIR:?}/${PATTERN_NAME}Segment"
-
-echo "Preparing temp directory for ${PATTERN_NAME}"
-rm -rf "${DATA_DIR}"
-rm -rf "${SEGMENT_DIR}"
-mkdir -p "${TEMP_DIR}"
-
-echo "Generating data for ${PATTERN_NAME} in ${DATA_DIR}"
-${ADMIN_PATH} GenerateData \
--numFiles 1 -numRecords 354780 -format csv \
--schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" \
--schemaAnnotationFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_generator.json" \
--outDir "$DATA_DIR"
-
-if [ ! -d "${DATA_DIR}" ]; then
- echo "Data generation failed. Aborting."
- exit 1
-fi
-
-echo "Creating segment for ${PATTERN_NAME} in ${SEGMENT_DIR}"
-${ADMIN_PATH} CreateSegment \
--tableName "${PATTERN_NAME}" -segmentName "${PATTERN_NAME}" -format CSV
-overwrite \
--schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" \
--dataDir "${DATA_DIR}" \
--outDir "${SEGMENT_DIR}" || exit 1
-
-if [ ! -d "${SEGMENT_DIR}" ]; then
- echo "Data generation failed. Aborting."
- exit 1
-fi
-
-echo "Adding table ${PATTERN_NAME}"
-${ADMIN_PATH} AddTable -exec \
--tableConfigFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_config.json" \
--schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" || exit 1
-
-echo "Uploading segment for ${PATTERN_NAME}"
-${ADMIN_PATH} UploadSegment \
--tableName "${PATTERN_NAME}" \
--segmentDir "${SEGMENT_DIR}" || exit 1
-
-echo "Succesfully applied PATTERN ${PATTERN_NAME}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
