Repository: kylin Updated Branches: refs/heads/KYLIN-2283 5957ee118 -> c35b6b50d
KYLIN-2283 fix CI Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c35b6b50 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c35b6b50 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c35b6b50 Branch: refs/heads/KYLIN-2283 Commit: c35b6b50da910667e4cec8a213a6be7fd7e4951e Parents: 5957ee1 Author: Yang Li <liy...@apache.org> Authored: Sat Dec 17 19:54:42 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sat Dec 17 19:54:42 2016 +0800 ---------------------------------------------------------------------- assembly/src/test/java/org/apache/kylin/job/DeployUtil.java | 2 +- .../org/apache/kylin/source/datagen/ColumnGenConfig.java | 5 ++++- .../org/apache/kylin/source/datagen/ColumnGenerator.java | 8 +++++--- .../org/apache/kylin/source/datagen/ModelDataGenerator.java | 2 +- .../localmeta/table/DEFAULT.TEST_KYLIN_FACT.json | 5 ++--- 5 files changed, 13 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c35b6b50/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index f65d2a4..8fc583d 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -139,7 +139,7 @@ public class DeployUtil { // data is generated according to cube descriptor and saved in resource store MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); DataModelDesc model = mgr.getDataModelDesc("test_kylin_inner_join_model_desc"); - ModelDataGenerator gen = new ModelDataGenerator(model, 1000); + ModelDataGenerator gen = new ModelDataGenerator(model, 10000); gen.generate(); } else { System.out.println("build normal cubes with provided dataset"); http://git-wip-us.apache.org/repos/asf/kylin/blob/c35b6b50/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java index 62da805..6387873 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java @@ -27,6 +27,7 @@ import org.apache.kylin.metadata.model.ColumnDesc; public class ColumnGenConfig { + public static final String FK = "FK"; public static final String ID = "ID"; public static final String RAND = "RAND"; public static final String $RANDOM = "${RANDOM}"; @@ -50,6 +51,7 @@ public class ColumnGenConfig { int cardinality; boolean genNull; double genNullPct; + String genNullStr; boolean order; boolean unique; @@ -65,7 +67,7 @@ public class ColumnGenConfig { List<String> pkValues = modelGen.getPkValuesIfIsFk(col); - if (pkValues != null) { + if (FK.equals(values.get(0)) || (values.get(0).isEmpty() && pkValues != null)) { isFK = true; values = pkValues; } else if (ID.equals(values.get(0))) { @@ -83,6 +85,7 @@ public class ColumnGenConfig { cardinality = Util.parseInt(config, "card", guessCardinality(col.getName())); genNull = Util.parseBoolean(config, "null", guessGenNull(col.getName())); genNullPct = Util.parseDouble(config, "nullpct", 0.01); + genNullStr = Util.parseString(config, "nullstr", "\\N"); // '\N' is null in hive order = Util.parseBoolean(config, "order", false); unique = Util.parseBoolean(config, "uniq", modelGen.isPK(col)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c35b6b50/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java index f171237..775f4fc 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java @@ -66,7 +66,7 @@ public class ColumnGenerator { } if (conf.genNull) { - result = new AddNullFilter(result, conf.genNullPct); + result = new AddNullFilter(result, conf.genNullPct, conf.genNullStr); } if (conf.order || conf.unique) { @@ -291,11 +291,13 @@ public class ColumnGenerator { private Iterator<String> input; private double nullPct; + private String nullStr; private Random rand; - public AddNullFilter(Iterator<String> input, double nullPct) { + public AddNullFilter(Iterator<String> input, double nullPct, String nullStr) { this.input = input; this.nullPct = nullPct; + this.nullStr = nullStr; this.rand = new Random(); } @@ -306,7 +308,7 @@ public class ColumnGenerator { @Override public String next() { - return rand.nextDouble() < nullPct || !input.hasNext() ? null : input.next(); + return rand.nextDouble() < nullPct || !input.hasNext() ? nullStr : input.next(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/c35b6b50/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java index 1319528..c325ab0 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java @@ -129,7 +129,7 @@ public class ModelDataGenerator { String v = colIters[c].next(); Preconditions.checkState(v == null || !v.contains(",")); - out.print(v == null ? "\\N" : v); // \N is null for hive + out.print(v); } out.print("\n"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c35b6b50/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json index 74eb045..1f3b183 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json @@ -11,7 +11,7 @@ "id" : "2", "name" : "CAL_DT", "datatype" : "date", - "data_gen" : "FK,order" + "data_gen" : "FK,order,null,nullstr=1970-01-01" }, { "id" : "3", "name" : "LSTG_FORMAT_NAME", @@ -28,8 +28,7 @@ }, { "id" : "6", "name" : "SLR_SEGMENT_CD", - "datatype" : "smallint", - "data_gen" : "FK,null" + "datatype" : "smallint" }, { "id" : "7", "name" : "PRICE",
