much slimer CI, ready to CI test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df27eb6b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df27eb6b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df27eb6b Branch: refs/heads/KYLIN-2294 Commit: df27eb6b5426c19873d4e9ed87910cb57ce4426c Parents: b8e03f2 Author: Li Yang <liy...@apache.org> Authored: Wed Dec 21 17:50:43 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Wed Dec 21 17:50:43 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/DeployUtil.java | 4 +- .../kylin/source/datagen/ColumnGenerator.java | 14 ++- .../kylin/provision/BuildCubeWithEngine.java | 89 ++++---------------- 3 files changed, 28 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/df27eb6b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index c8b0d43..e56c6bc 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -130,7 +130,7 @@ public class DeployUtil { static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES }; - public static void prepareTestDataForNormalCubes(String cubeName) throws Exception { + public static void prepareTestDataForNormalCubes(String modelName) throws Exception { boolean buildCubeUsingProvidedData = Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData")); if (!buildCubeUsingProvidedData) { @@ -138,7 +138,7 @@ public class DeployUtil { // data is generated according to cube descriptor and saved in resource store MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - DataModelDesc model = mgr.getDataModelDesc("test_kylin_inner_join_model_desc"); + DataModelDesc model = mgr.getDataModelDesc(modelName); ModelDataGenerator gen = new ModelDataGenerator(model, 10000); gen.generate(); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/df27eb6b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java index 775f4fc..7c54714 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java @@ -250,8 +250,8 @@ public class ColumnGenerator { public String next() { if (values.isEmpty()) return null; - else - return values.get(rand.nextInt(values.size())); + + return values.get(rand.nextInt(values.size())); } } @@ -308,7 +308,15 @@ public class ColumnGenerator { @Override public String next() { - return rand.nextDouble() < nullPct || !input.hasNext() ? nullStr : input.next(); + String r = nullStr; + if (input.hasNext()) { + r = input.next(); + } + + if (rand.nextDouble() < nullPct) { + r = nullStr; + } + return r; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/df27eb6b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index bfbeb70..66a217a 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -161,7 +161,7 @@ public class BuildCubeWithEngine { } public void build() throws Exception { - DeployUtil.prepareTestDataForNormalCubes("test_kylin_cube_with_slr_empty"); + DeployUtil.prepareTestDataForNormalCubes("ci_left_join_model"); KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(1.0f); testInner(); testLeft(); @@ -184,12 +184,12 @@ public class BuildCubeWithEngine { } private void testInner() throws Exception { - String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" }; + String[] testCase = new String[] { "testInnerJoinCube" }; runTestAndAssertSucceed(testCase); } private void testLeft() throws Exception { - String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" }; + String[] testCase = new String[] { "testLeftJoinCube" }; runTestAndAssertSucceed(testCase); } @@ -243,47 +243,24 @@ public class BuildCubeWithEngine { @SuppressWarnings("unused") // called by reflection - private boolean testInnerJoinCubeWithSlr() throws Exception { - final String cubeName = "test_kylin_cube_with_slr_empty"; - clearSegment(cubeName); - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = 0; - long date2 = f.parse("2015-01-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); - - if (fastBuildMode) { - return buildSegment(cubeName, date1, date3); - } else { - if (buildSegment(cubeName, date1, date2) == true) { - return buildSegment(cubeName, date2, date3);//empty segment - } - } - return false; - } - - @SuppressWarnings("unused") - // called by reflection - private boolean testLeftJoinCubeWithSlr() throws Exception { - String cubeName = "test_kylin_cube_with_slr_left_join_empty"; + private boolean testLeftJoinCube() throws Exception { + String cubeName = "ci_left_join_cube"; clearSegment(cubeName); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); + long date1 = 0; long date2 = f.parse("2013-01-01").getTime(); long date3 = f.parse("2013-07-01").getTime(); long date4 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); if (fastBuildMode) { return buildSegment(cubeName, date1, date4); } else { if (buildSegment(cubeName, date1, date2) == true) { if (buildSegment(cubeName, date2, date3) == true) { if (buildSegment(cubeName, date3, date4) == true) { - return mergeSegment(cubeName, date1, date3);//don't merge all segments + return mergeSegment(cubeName, date1, date3); // don't merge all segments } } } @@ -293,61 +270,25 @@ public class BuildCubeWithEngine { @SuppressWarnings("unused") // called by reflection - private boolean testInnerJoinCubeWithoutSlr() throws Exception { + private boolean testInnerJoinCube() throws Exception { - final String cubeName = "test_kylin_cube_without_slr_empty"; + String cubeName = "ci_inner_join_cube"; clearSegment(cubeName); + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); long date1 = 0; - long date2 = f.parse("2013-01-01").getTime(); - long date3 = f.parse("2013-07-01").getTime(); - long date4 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); + long date2 = f.parse("2022-07-01").getTime(); + long date3 = f.parse("2023-01-01").getTime(); if (fastBuildMode) { - return buildSegment(cubeName, date1, date4); - } else { - if (buildSegment(cubeName, date1, date2) == true) { - if (buildSegment(cubeName, date2, date3) == true) { - if (buildSegment(cubeName, date3, date4) == true) { - return mergeSegment(cubeName, date1, date3);//don't merge all segments - } - } - } - } - return false; - - } - - @SuppressWarnings("unused") - // called by reflection - private boolean testLeftJoinCubeWithoutSlr() throws Exception { - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - List<String> result = Lists.newArrayList(); - final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; - clearSegment(cubeName); - - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); - long date2 = f.parse("2012-06-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); - long date4 = f.parse("2023-01-01").getTime(); - - if (fastBuildMode) { - return buildSegment(cubeName, date1, date4); + return buildSegment(cubeName, date1, date3); } else { - if (buildSegment(cubeName, date1, date2) == true) { - if (buildSegment(cubeName, date2, date3) == true) { - if (buildSegment(cubeName, date3, date4) == true) { //empty segment - return mergeSegment(cubeName, date1, date3);//don't merge all segments - } - } + if (buildSegment(cubeName, date1, date2) == true) { // all-in-one build + return buildSegment(cubeName, date2, date3); // empty segment } } - return false; - } private void clearSegment(String cubeName) throws Exception {
