Repository: kylin Updated Branches: refs/heads/KYLIN-2295 [created] 1e7665de5
KYLIN-2295 let us try Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e7665de Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e7665de Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e7665de Branch: refs/heads/KYLIN-2295 Commit: 1e7665de5a83b0ea619f954a8f245f6ce85896ff Parents: d1175d2 Author: Yang Li <liy...@apache.org> Authored: Sun Dec 18 20:24:25 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sun Dec 18 20:24:25 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/DeployUtil.java | 16 +- .../source/datagen/ModelDataGenerator.java | 8 + .../kylin/source/datagen/TableGenConfig.java | 5 +- ...t_kylin_cube_with_view_inner_join_empty.json | 10 - ...st_kylin_cube_with_view_left_join_empty.json | 10 - ...st_kylin_cube_with_view_inner_join_desc.json | 169 -------- ...est_kylin_cube_with_view_left_join_desc.json | 169 -------- .../localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv | 8 - .../data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv | 8 + .../test_kylin_inner_join_view_model_desc.json | 119 ------ .../test_kylin_left_join_view_model_desc.json | 119 ------ .../localmeta/project/default.json | 14 +- .../table/EDW.TEST_SELLER_TYPE_DIM.json | 2 +- .../table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json | 43 ++ .../localmeta/table/EDW.V_TEST_CAL_DT.json | 409 ------------------- .../kylin/provision/BuildCubeWithEngine.java | 83 +--- 16 files changed, 93 insertions(+), 1099 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index 8fc583d..7579dfc 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -122,13 +122,13 @@ public class DeployUtil { // ============================================================================ static final String TABLE_CAL_DT = "edw.test_cal_dt"; - static final String VIEW_CAL_DT = "edw.v_test_cal_dt"; static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings"; static final String TABLE_KYLIN_FACT = "default.test_kylin_fact"; - static final String TABLE_SELLER_TYPE_DIM = "edw.test_seller_type_dim"; + static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim"; + static final String TABLE_SELLER_TYPE_DIM_TABLE = "edw.test_seller_type_dim_table"; static final String TABLE_SITES = "edw.test_sites"; - static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM, TABLE_SITES }; + static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES }; public static void prepareTestDataForNormalCubes(String cubeName) throws Exception { @@ -230,7 +230,7 @@ public class DeployUtil { hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM.toUpperCase()))); + hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase()))); // load data to hive tables @@ -238,11 +238,11 @@ public class DeployUtil { hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir)); hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir)); hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM, tableFileDir)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM_TABLE, tableFileDir)); hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir)); final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); - hiveCmdBuilder.addStatements(generateCreateViewHql(VIEW_CAL_DT, metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); + hiveCmdBuilder.addStatements(generateCreateViewHql(VIEW_SELLER_TYPE_DIM, TABLE_SELLER_TYPE_DIM_TABLE)); config().getCliCommandExecutor().execute(hiveCmdBuilder.build()); } @@ -274,12 +274,12 @@ public class DeployUtil { return new String[] { dropsql, ddl.toString() }; } - private static String[] generateCreateViewHql(String viewName, TableDesc tableDesc) { + private static String[] generateCreateViewHql(String viewName, String tableName) { String dropsql = "DROP VIEW IF EXISTS " + viewName + ";"; StringBuilder ddl = new StringBuilder(); - ddl.append("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableDesc.getIdentity() + ";\n"); + ddl.append("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableName + ";\n"); return new String[] { dropsql, ddl.toString() }; } http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java index c325ab0..7f5d500 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java @@ -166,6 +166,9 @@ public class ModelDataGenerator { private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { + if (t.isView()) + continue; + out.print("DROP TABLE IF EXISTS " + t.getIdentity() + ";\n"); out.print("CREATE TABLE " + t.getIdentity() + "(" + "\n"); @@ -198,6 +201,11 @@ public class ModelDataGenerator { private void generateLoadDataDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { + if (t.isView()) { + out.print("-- " + t.getIdentity() + " is view \n"); + continue; + } + out.print("LOAD DATA LOCAL INPATH '" + t.getIdentity() + ".csv' OVERWRITE INTO TABLE " + t.getIdentity() + ";\n"); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java index be948c1..a0f19e7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java @@ -34,7 +34,10 @@ public class TableGenConfig { dataGen = ""; } - if (dataGen == null) + if (dataGen == null || "no".equals(dataGen) || "false".equals(dataGen) || "skip".equals(dataGen)) + return; + + if (table.isView()) return; needGen = true; http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json deleted file mode 100644 index 6215174..0000000 --- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "uuid" : "ffff3e80-41be-49a5-90ca-9fb7294db186", - "last_modified" : 0, - "name" : "test_kylin_cube_with_view_inner_join_empty", - "owner" : null, - "descriptor" : "test_kylin_cube_with_view_inner_join_desc", - "segments" : [ ], - "status" : "DISABLED", - "create_time" : null -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json deleted file mode 100644 index 62b7432..0000000 --- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "uuid" : "eeee3e80-41be-49a5-90ca-9fb7294db186", - "last_modified" : 0, - "name" : "test_kylin_cube_with_view_left_join_empty", - "owner" : null, - "descriptor" : "test_kylin_cube_with_view_left_join_desc", - "segments" : [ ], - "status" : "DISABLED", - "create_time" : null -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json deleted file mode 100644 index d4c64b5..0000000 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json +++ /dev/null @@ -1,169 +0,0 @@ -{ - "uuid" : "9876b7a8-3929-4dff-b59d-2100aadc8dbf", - "name" : "test_kylin_cube_with_view_inner_join_desc", - "description" : null, - "dimensions" : [ { - "name" : "CAL_DT", - "table" : "EDW.V_TEST_CAL_DT", - "column" : "{FK}", - "derived" : [ "WEEK_BEG_DT" ] - }, { - "name" : "CATEGORY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", - "column" : "{FK}", - "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", - "column" : "META_CATEG_NAME", - "derived" : null - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL2_NAME", - "derived" : null - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL3_NAME", - "derived" : null - }, { - "name" : "LSTG_FORMAT_NAME", - "table" : "DEFAULT.TEST_KYLIN_FACT", - "column" : "LSTG_FORMAT_NAME", - "derived" : null - }, { - "name" : "SITE_ID", - "table" : "EDW.TEST_SITES", - "column" : "{FK}", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "name" : "SELLER_TYPE_CD", - "table" : "EDW.TEST_SELLER_TYPE_DIM", - "column" : "{FK}", - "derived" : [ "SELLER_TYPE_DESC" ] - } ], - "measures" : [ { - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null - }, { - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null - }, { - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null - }, { - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1", - "next_parameter" : null - }, - "returntype" : "bigint" - }, - "dependent_measure_ref" : null - }, { - "name" : "ITEM_COUNT_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "ITEM_COUNT", - "next_parameter" : null - }, - "returntype" : "bigint" - }, - "dependent_measure_ref" : null - }], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "cal_dt", - "encoding" : "dict" - }, { - "column" : "leaf_categ_id", - "encoding" : "dict" - }, { - "column" : "meta_categ_name", - "encoding" : "dict" - }, { - "column" : "categ_lvl2_name", - "encoding" : "dict" - }, { - "column" : "categ_lvl3_name", - "encoding" : "dict" - }, { - "column" : "lstg_format_name", - "encoding" : "dict" - }, { - "column" : "lstg_site_id", - "encoding" : "dict" - }, { - "column" : "slr_segment_cd", - "encoding" : "dict" - } ] - }, - "signature" : null, - "last_modified" : 1448959801311, - "model_name" : "test_kylin_inner_join_view_model_desc", - "null_string" : null, - "hbase_mapping" : { - "column_family" : [ { - "name" : "f1", - "columns" : [ { - "qualifier" : "m", - "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ] - } ] - }] - }, - "aggregation_groups" : [ { - "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"], - "select_rule" : { - "hierarchy_dims" : [ ], - "mandatory_dims" : [ "cal_dt" ], - "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ] - } - }, { - "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ], - "select_rule" : { - "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [ "cal_dt" ], - "joint_dims" : [ ] - } - } ], - "notify_list" : null, - "status_need_notify" : [ ], - "auto_merge_time_ranges" : null, - "retention_range" : 0, - "engine_type" : 2, - "storage_type" : 2, - "partition_date_start": 0 -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json deleted file mode 100644 index 0388c0e..0000000 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json +++ /dev/null @@ -1,169 +0,0 @@ -{ - "uuid" : "6789b7a8-3929-4dff-b59d-2100aadc8dbf", - "name" : "test_kylin_cube_with_view_left_join_desc", - "description" : null, - "dimensions" : [ { - "name" : "CAL_DT", - "table" : "EDW.V_TEST_CAL_DT", - "column" : "{FK}", - "derived" : [ "WEEK_BEG_DT" ] - }, { - "name" : "CATEGORY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", - "column" : "{FK}", - "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", - "column" : "META_CATEG_NAME", - "derived" : null - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL2_NAME", - "derived" : null - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL3_NAME", - "derived" : null - }, { - "name" : "LSTG_FORMAT_NAME", - "table" : "DEFAULT.TEST_KYLIN_FACT", - "column" : "LSTG_FORMAT_NAME", - "derived" : null - }, { - "name" : "SITE_ID", - "table" : "EDW.TEST_SITES", - "column" : "{FK}", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "name" : "SELLER_TYPE_CD", - "table" : "EDW.TEST_SELLER_TYPE_DIM", - "column" : "{FK}", - "derived" : [ "SELLER_TYPE_DESC" ] - } ], - "measures" : [ { - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null - }, { - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null - }, { - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null - }, { - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1", - "next_parameter" : null - }, - "returntype" : "bigint" - }, - "dependent_measure_ref" : null - }, { - "name" : "ITEM_COUNT_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "ITEM_COUNT", - "next_parameter" : null - }, - "returntype" : "bigint" - }, - "dependent_measure_ref" : null - }], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "cal_dt", - "encoding" : "dict" - }, { - "column" : "leaf_categ_id", - "encoding" : "dict" - }, { - "column" : "meta_categ_name", - "encoding" : "dict" - }, { - "column" : "categ_lvl2_name", - "encoding" : "dict" - }, { - "column" : "categ_lvl3_name", - "encoding" : "dict" - }, { - "column" : "lstg_format_name", - "encoding" : "dict" - }, { - "column" : "lstg_site_id", - "encoding" : "dict" - }, { - "column" : "slr_segment_cd", - "encoding" : "dict" - } ] - }, - "signature" : null, - "last_modified" : 1448959801311, - "model_name" : "test_kylin_left_join_view_model_desc", - "null_string" : null, - "hbase_mapping" : { - "column_family" : [ { - "name" : "f1", - "columns" : [ { - "qualifier" : "m", - "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ] - } ] - }] - }, - "aggregation_groups" : [ { - "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"], - "select_rule" : { - "hierarchy_dims" : [ ], - "mandatory_dims" : [ "cal_dt" ], - "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ] - } - }, { - "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ], - "select_rule" : { - "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [ "cal_dt" ], - "joint_dims" : [ ] - } - } ], - "notify_list" : null, - "status_need_notify" : [ ], - "auto_merge_time_ranges" : null, - "retention_range" : 0, - "engine_type" : 2, - "storage_type" : 2, - "partition_date_start": 0 -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv b/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv deleted file mode 100644 index 73abe26..0000000 --- a/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv +++ /dev/null @@ -1,8 +0,0 @@ -16,Consumer-Other,16,102,Consumer,2012-08-09,USER_P,,USER_A -5,Occasional,5,102,Consumer,2010-05-11,USER_H,2011-03-01 11:30:52,USER_A -15,Professional-Other,15,101,Professional,2012-08-09,USER_P,,USER_A --99,Not Applicable,-99,-99,Not Applicable,2010-05-11,USER_H,,USER_A -14,Regular,14,102,Consumer,2011-03-01,USER_H,,USER_A -12,Merchant,12,101,Professional,2011-03-01,USER_H,,USER_A -11,Large Merchant,11,101,Professional,2011-03-01,USER_H,,USER_A -13,Entrepreneur,13,101,Professional,2011-03-01,USER_H,,USER_A http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv b/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv new file mode 100644 index 0000000..d653ae2 --- /dev/null +++ b/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv @@ -0,0 +1,8 @@ +16,Consumer-Other,16,102,Consumer,2012-08-09,USER_P,,USER_A +5,Occasional,5,102,Consumer,2010-05-11,USER_H,2011-03-01 11:30:52,USER_A +15,Professional-Other,15,101,Professional,2012-08-09,USER_P,,USER_A +-99,Not Applicable,-99,-99,Not Applicable,2010-05-11,USER_H,,USER_A +14,Regular,14,102,Consumer,2011-03-01,USER_H,,USER_A +12,Merchant,12,101,Professional,2011-03-01,USER_H,,USER_A +11,Large Merchant,11,101,Professional,2011-03-01,USER_H,,USER_A +13,Entrepreneur,13,101,Professional,2011-03-01,USER_H,,USER_A http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json deleted file mode 100644 index d7b7227..0000000 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "uuid": "8b184ee2-1ccb-4b07-a38e-4c298563e0f7", - "name": "test_kylin_inner_join_view_model_desc", - "lookups": [ - { - "table": "EDW.V_TEST_CAL_DT", - "join": { - "type": "inner", - "primary_key": [ - "CAL_DT" - ], - "foreign_key": [ - "CAL_DT" - ] - } - }, - { - "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", - "join": { - "type": "inner", - "primary_key": [ - "LEAF_CATEG_ID", - "SITE_ID" - ], - "foreign_key": [ - "LEAF_CATEG_ID", - "LSTG_SITE_ID" - ] - } - }, - { - "table": "EDW.TEST_SITES", - "join": { - "type": "inner", - "primary_key": [ - "SITE_ID" - ], - "foreign_key": [ - "LSTG_SITE_ID" - ] - } - }, - { - "table": "EDW.TEST_SELLER_TYPE_DIM", - "join": { - "type": "inner", - "primary_key": [ - "SELLER_TYPE_CD" - ], - "foreign_key": [ - "SLR_SEGMENT_CD" - ] - } - } - ], - "dimensions": [ - { - "table": "default.test_kylin_fact", - "columns": [ - "TRANS_ID", - "CAL_DT", - "lstg_format_name", - "LSTG_SITE_ID", - "SLR_SEGMENT_CD", - "SELLER_ID" - ] - }, - { - "table": "default.test_category_groupings", - "columns": [ - "leaf_categ_id", - "site_id", - "USER_DEFINED_FIELD1", - "USER_DEFINED_FIELD3", - "UPD_DATE", - "UPD_USER", - "meta_categ_name", - "categ_lvl2_name", - "categ_lvl3_name" - ] - }, - { - "table": "edw.test_sites", - "columns": [ - "site_id", - "site_name", - "cre_user" - ] - }, - { - "table": "edw.test_seller_type_dim", - "columns": [ - "seller_type_cd", - "seller_type_desc" - ] - }, - { - "table": "edw.v_test_cal_dt", - "columns": [ - "cal_dt", - "week_beg_dt" - ] - } - ], - "metrics": [ - "PRICE", - "ITEM_COUNT", - "SELLER_ID", - "USER_ID" - ], - "last_modified": 1422435345352, - "fact_table": "DEFAULT.TEST_KYLIN_FACT", - "filter_condition": null, - "partition_desc": { - "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt", - "partition_date_start": 0, - "partition_type": "APPEND" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json deleted file mode 100644 index 2ff55d3..0000000 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "uuid": "9c184ee2-1ccb-4b07-a38e-4c298563e0f7", - "name": "test_kylin_left_join_view_model_desc", - "lookups": [ - { - "table": "EDW.V_TEST_CAL_DT", - "join": { - "type": "left", - "primary_key": [ - "CAL_DT" - ], - "foreign_key": [ - "CAL_DT" - ] - } - }, - { - "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", - "join": { - "type": "left", - "primary_key": [ - "LEAF_CATEG_ID", - "SITE_ID" - ], - "foreign_key": [ - "LEAF_CATEG_ID", - "LSTG_SITE_ID" - ] - } - }, - { - "table": "EDW.TEST_SITES", - "join": { - "type": "left", - "primary_key": [ - "SITE_ID" - ], - "foreign_key": [ - "LSTG_SITE_ID" - ] - } - }, - { - "table": "EDW.TEST_SELLER_TYPE_DIM", - "join": { - "type": "left", - "primary_key": [ - "SELLER_TYPE_CD" - ], - "foreign_key": [ - "SLR_SEGMENT_CD" - ] - } - } - ], - "dimensions": [ - { - "table": "default.test_kylin_fact", - "columns": [ - "TRANS_ID", - "CAL_DT", - "lstg_format_name", - "LSTG_SITE_ID", - "SLR_SEGMENT_CD", - "SELLER_ID" - ] - }, - { - "table": "default.test_category_groupings", - "columns": [ - "leaf_categ_id", - "site_id", - "USER_DEFINED_FIELD1", - "USER_DEFINED_FIELD3", - "UPD_DATE", - "UPD_USER", - "meta_categ_name", - "categ_lvl2_name", - "categ_lvl3_name" - ] - }, - { - "table": "edw.test_sites", - "columns": [ - "site_id", - "site_name", - "cre_user" - ] - }, - { - "table": "edw.test_seller_type_dim", - "columns": [ - "seller_type_cd", - "seller_type_desc" - ] - }, - { - "table": "edw.v_test_cal_dt", - "columns": [ - "cal_dt", - "week_beg_dt" - ] - } - ], - "metrics": [ - "PRICE", - "ITEM_COUNT", - "SELLER_ID", - "USER_ID" - ], - "last_modified": 1422435345352, - "fact_table": "DEFAULT.TEST_KYLIN_FACT", - "filter_condition": null, - "partition_desc": { - "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt", - "partition_date_start": 0, - "partition_type": "APPEND" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/project/default.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index d24a22a..5eefce7 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -32,16 +32,6 @@ "type": "HYBRID", "realization": "test_kylin_hybrid_inner_join" }, - { - "name": "test_kylin_cube_with_view_left_join_empty", - "type": "CUBE", - "realization": "test_kylin_cube_with_view_left_join_empty" - }, - { - "name": "test_kylin_cube_with_view_inner_join_empty", - "type": "CUBE", - "realization": "test_kylin_cube_with_view_inner_join_empty" - }, { "name": "ssb", "type": "CUBE", @@ -51,18 +41,16 @@ "tables": [ "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT", - "EDW.V_TEST_CAL_DT", "DEFAULT.TEST_CATEGORY_GROUPINGS", "EDW.TEST_SITES", "EDW.TEST_SELLER_TYPE_DIM", + "EDW.TEST_SELLER_TYPE_DIM_TABLE", "DEFAULT.STREAMING_TABLE" ], "models": [ "ssb", "test_kylin_inner_join_model_desc", - "test_kylin_inner_join_view_model_desc", "test_kylin_left_join_model_desc", - "test_kylin_left_join_view_model_desc", "test_streaming_table_model_desc" ], "override_kylin_properties" :{ http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json index eec48ea..a2be051 100644 --- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json +++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json @@ -1,5 +1,4 @@ { - "uuid" : "9ecc90c4-55df-436f-8602-2fbd4bca72e1", "name" : "TEST_SELLER_TYPE_DIM", "columns" : [ { @@ -40,5 +39,6 @@ "datatype" : "string" } ], "database" : "edw", + "table_type" : "VIRTUAL_VIEW", "last_modified" : 0 } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json new file mode 100644 index 0000000..3c5749b --- /dev/null +++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json @@ -0,0 +1,43 @@ +{ + "uuid" : "addc20c4-55df-436f-8602-2fbd4bca2910", + "name" : "TEST_SELLER_TYPE_DIM_TABLE", + "columns" : [ { + "id" : "1", + "name" : "SELLER_TYPE_CD", + "datatype" : "smallint" + }, { + "id" : "2", + "name" : "SELLER_TYPE_DESC", + "datatype" : "string" + }, { + "id" : "3", + "name" : "GLBL_RPRT_SLR_SGMNT_CD", + "datatype" : "tinyint" + }, { + "id" : "4", + "name" : "SELLER_GROUP_CD", + "datatype" : "tinyint" + }, { + "id" : "5", + "name" : "SELLER_GROUP_DESC", + "datatype" : "string" + }, { + "id" : "6", + "name" : "DIM_CRE_DATE", + "datatype" : "string" + }, { + "id" : "7", + "name" : "DIM_CRE_USER", + "datatype" : "string" + }, { + "id" : "8", + "name" : "DIM_UPD_DATE", + "datatype" : "string" + }, { + "id" : "9", + "name" : "DIM_UPD_USER", + "datatype" : "string" + } ], + "database" : "edw", + "last_modified" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json deleted file mode 100644 index 9298c7f..0000000 --- a/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json +++ /dev/null @@ -1,409 +0,0 @@ -{ - - "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a", - "name" : "V_TEST_CAL_DT", - "columns" : [ { - "id" : "1", - "name" : "CAL_DT", - "datatype" : "date" - }, { - "id" : "2", - "name" : "YEAR_BEG_DT", - "datatype" : "date" - }, { - "id" : "3", - "name" : "QTR_BEG_DT", - "datatype" : "date" - }, { - "id" : "4", - "name" : "MONTH_BEG_DT", - "datatype" : "date" - }, { - "id" : "5", - "name" : "WEEK_BEG_DT", - "datatype" : "date" - }, { - "id" : "6", - "name" : "AGE_FOR_YEAR_ID", - "datatype" : "smallint" - }, { - "id" : "7", - "name" : "AGE_FOR_QTR_ID", - "datatype" : "smallint" - }, { - "id" : "8", - "name" : "AGE_FOR_MONTH_ID", - "datatype" : "smallint" - }, { - "id" : "9", - "name" : "AGE_FOR_WEEK_ID", - "datatype" : "smallint" - }, { - "id" : "10", - "name" : "AGE_FOR_DT_ID", - "datatype" : "smallint" - }, { - "id" : "11", - "name" : "AGE_FOR_RTL_YEAR_ID", - "datatype" : "smallint" - }, { - "id" : "12", - "name" : "AGE_FOR_RTL_QTR_ID", - "datatype" : "smallint" - }, { - "id" : "13", - "name" : "AGE_FOR_RTL_MONTH_ID", - "datatype" : "smallint" - }, { - "id" : "14", - "name" : "AGE_FOR_RTL_WEEK_ID", - "datatype" : "smallint" - }, { - "id" : "15", - "name" : "AGE_FOR_CS_WEEK_ID", - "datatype" : "smallint" - }, { - "id" : "16", - "name" : "DAY_OF_CAL_ID", - "datatype" : "int" - }, { - "id" : "17", - "name" : "DAY_OF_YEAR_ID", - "datatype" : "smallint" - }, { - "id" : "18", - "name" : "DAY_OF_QTR_ID", - "datatype" : "smallint" - }, { - "id" : "19", - "name" : "DAY_OF_MONTH_ID", - "datatype" : "smallint" - }, { - "id" : "20", - "name" : "DAY_OF_WEEK_ID", - "datatype" : "int" - }, { - "id" : "21", - "name" : "WEEK_OF_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "22", - "name" : "WEEK_OF_CAL_ID", - "datatype" : "int" - }, { - "id" : "23", - "name" : "MONTH_OF_QTR_ID", - "datatype" : "tinyint" - }, { - "id" : "24", - "name" : "MONTH_OF_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "25", - "name" : "MONTH_OF_CAL_ID", - "datatype" : "smallint" - }, { - "id" : "26", - "name" : "QTR_OF_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "27", - "name" : "QTR_OF_CAL_ID", - "datatype" : "smallint" - }, { - "id" : "28", - "name" : "YEAR_OF_CAL_ID", - "datatype" : "smallint" - }, { - "id" : "29", - "name" : "YEAR_END_DT", - "datatype" : "string" - }, { - "id" : "30", - "name" : "QTR_END_DT", - "datatype" : "string" - }, { - "id" : "31", - "name" : "MONTH_END_DT", - "datatype" : "string" - }, { - "id" : "32", - "name" : "WEEK_END_DT", - "datatype" : "string" - }, { - "id" : "33", - "name" : "CAL_DT_NAME", - "datatype" : "string" - }, { - "id" : "34", - "name" : "CAL_DT_DESC", - "datatype" : "string" - }, { - "id" : "35", - "name" : "CAL_DT_SHORT_NAME", - "datatype" : "string" - }, { - "id" : "36", - "name" : "YTD_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "37", - "name" : "QTD_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "38", - "name" : "MTD_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "39", - "name" : "WTD_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "40", - "name" : "SEASON_BEG_DT", - "datatype" : "string" - }, { - "id" : "41", - "name" : "DAY_IN_YEAR_COUNT", - "datatype" : "smallint" - }, { - "id" : "42", - "name" : "DAY_IN_QTR_COUNT", - "datatype" : "tinyint" - }, { - "id" : "43", - "name" : "DAY_IN_MONTH_COUNT", - "datatype" : "tinyint" - }, { - "id" : "44", - "name" : "DAY_IN_WEEK_COUNT", - "datatype" : "tinyint" - }, { - "id" : "45", - "name" : "RTL_YEAR_BEG_DT", - "datatype" : "string" - }, { - "id" : "46", - "name" : "RTL_QTR_BEG_DT", - "datatype" : "string" - }, { - "id" : "47", - "name" : "RTL_MONTH_BEG_DT", - "datatype" : "string" - }, { - "id" : "48", - "name" : "RTL_WEEK_BEG_DT", - "datatype" : "string" - }, { - "id" : "49", - "name" : "CS_WEEK_BEG_DT", - "datatype" : "string" - }, { - "id" : "50", - "name" : "CAL_DATE", - "datatype" : "string" - }, { - "id" : "51", - "name" : "DAY_OF_WEEK", - "datatype" : "string" - }, { - "id" : "52", - "name" : "MONTH_ID", - "datatype" : "string" - }, { - "id" : "53", - "name" : "PRD_DESC", - "datatype" : "string" - }, { - "id" : "54", - "name" : "PRD_FLAG", - "datatype" : "string" - }, { - "id" : "55", - "name" : "PRD_ID", - "datatype" : "string" - }, { - "id" : "56", - "name" : "PRD_IND", - "datatype" : "string" - }, { - "id" : "57", - "name" : "QTR_DESC", - "datatype" : "string" - }, { - "id" : "58", - "name" : "QTR_ID", - "datatype" : "string" - }, { - "id" : "59", - "name" : "QTR_IND", - "datatype" : "string" - }, { - "id" : "60", - "name" : "RETAIL_WEEK", - "datatype" : "string" - }, { - "id" : "61", - "name" : "RETAIL_YEAR", - "datatype" : "string" - }, { - "id" : "62", - "name" : "RETAIL_START_DATE", - "datatype" : "string" - }, { - "id" : "63", - "name" : "RETAIL_WK_END_DATE", - "datatype" : "string" - }, { - "id" : "64", - "name" : "WEEK_IND", - "datatype" : "string" - }, { - "id" : "65", - "name" : "WEEK_NUM_DESC", - "datatype" : "string" - }, { - "id" : "66", - "name" : "WEEK_BEG_DATE", - "datatype" : "string" - }, { - "id" : "67", - "name" : "WEEK_END_DATE", - "datatype" : "string" - }, { - "id" : "68", - "name" : "WEEK_IN_YEAR_ID", - "datatype" : "string" - }, { - "id" : "69", - "name" : "WEEK_ID", - "datatype" : "string" - }, { - "id" : "70", - "name" : "WEEK_BEG_END_DESC_MDY", - "datatype" : "string" - }, { - "id" : "71", - "name" : "WEEK_BEG_END_DESC_MD", - "datatype" : "string" - }, { - "id" : "72", - "name" : "YEAR_ID", - "datatype" : "string" - }, { - "id" : "73", - "name" : "YEAR_IND", - "datatype" : "string" - }, { - "id" : "74", - "name" : "CAL_DT_MNS_1YEAR_DT", - "datatype" : "string" - }, { - "id" : "75", - "name" : "CAL_DT_MNS_2YEAR_DT", - "datatype" : "string" - }, { - "id" : "76", - "name" : "CAL_DT_MNS_1QTR_DT", - "datatype" : "string" - }, { - "id" : "77", - "name" : "CAL_DT_MNS_2QTR_DT", - "datatype" : "string" - }, { - "id" : "78", - "name" : "CAL_DT_MNS_1MONTH_DT", - "datatype" : "string" - }, { - "id" : "79", - "name" : "CAL_DT_MNS_2MONTH_DT", - "datatype" : "string" - }, { - "id" : "80", - "name" : "CAL_DT_MNS_1WEEK_DT", - "datatype" : "string" - }, { - "id" : "81", - "name" : "CAL_DT_MNS_2WEEK_DT", - "datatype" : "string" - }, { - "id" : "82", - "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "83", - "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "84", - "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "85", - "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "86", - "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "87", - "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "88", - "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND", - "datatype" : "tinyint" - }, { - "id" : "89", - "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND", - "datatype" : "tinyint" - }, { - "id" : "90", - "name" : "RTL_MONTH_OF_RTL_YEAR_ID", - "datatype" : "string" - }, { - "id" : "91", - "name" : "RTL_QTR_OF_RTL_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "92", - "name" : "RTL_WEEK_OF_RTL_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "93", - "name" : "SEASON_OF_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "94", - "name" : "YTM_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "95", - "name" : "YTQ_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "96", - "name" : "YTW_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "97", - "name" : "V_CAL_DT_CRE_DATE", - "datatype" : "string" - }, { - "id" : "98", - "name" : "V_CAL_DT_CRE_USER", - "datatype" : "string" - }, { - "id" : "99", - "name" : "V_CAL_DT_UPD_DATE", - "datatype" : "string" - }, { - "id" : "100", - "name" : "V_CAL_DT_UPD_USER", - "datatype" : "string" - } ], - "database" : "edw", - "table_type" : "VIRTUAL_VIEW", - "last_modified" : 0 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/1e7665de/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index 67b62d5..bfbeb70 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -165,7 +165,6 @@ public class BuildCubeWithEngine { KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(1.0f); testInner(); testLeft(); - testViewAsLookup(); KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(0.0f); } @@ -194,11 +193,6 @@ public class BuildCubeWithEngine { runTestAndAssertSucceed(testCase); } - private void testViewAsLookup() throws Exception { - String[] testCase = new String[] { "testInnerJoinCubeWithView", "testLeftJoinCubeWithView" }; - runTestAndAssertSucceed(testCase); - } - private void runTestAndAssertSucceed(String[] testCase) throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(testCase.length); final CountDownLatch countDownLatch = new CountDownLatch(testCase.length); @@ -271,18 +265,18 @@ public class BuildCubeWithEngine { @SuppressWarnings("unused") // called by reflection - private Boolean testInnerJoinCubeWithoutSlr() throws Exception { - - final String cubeName = "test_kylin_cube_without_slr_empty"; + private boolean testLeftJoinCubeWithSlr() throws Exception { + String cubeName = "test_kylin_cube_with_slr_left_join_empty"; clearSegment(cubeName); + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = 0; + long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); long date2 = f.parse("2013-01-01").getTime(); long date3 = f.parse("2013-07-01").getTime(); long date4 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); + List<String> result = Lists.newArrayList(); if (fastBuildMode) { return buildSegment(cubeName, date1, date4); } else { @@ -295,96 +289,63 @@ public class BuildCubeWithEngine { } } return false; - } @SuppressWarnings("unused") // called by reflection - private boolean testLeftJoinCubeWithoutSlr() throws Exception { + private boolean testInnerJoinCubeWithoutSlr() throws Exception { + + final String cubeName = "test_kylin_cube_without_slr_empty"; + clearSegment(cubeName); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); + long date1 = 0; + long date2 = f.parse("2013-01-01").getTime(); + long date3 = f.parse("2013-07-01").getTime(); + long date4 = f.parse("2022-01-01").getTime(); List<String> result = Lists.newArrayList(); - final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; - clearSegment(cubeName); - - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); - long date2 = f.parse("2012-06-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); - long date4 = f.parse("2023-01-01").getTime(); if (fastBuildMode) { return buildSegment(cubeName, date1, date4); } else { if (buildSegment(cubeName, date1, date2) == true) { if (buildSegment(cubeName, date2, date3) == true) { - if (buildSegment(cubeName, date3, date4) == true) { //empty segment + if (buildSegment(cubeName, date3, date4) == true) { return mergeSegment(cubeName, date1, date3);//don't merge all segments } } } } - return false; } @SuppressWarnings("unused") // called by reflection - private boolean testLeftJoinCubeWithView() throws Exception { - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - List<String> result = Lists.newArrayList(); - final String cubeName = "test_kylin_cube_with_view_left_join_empty"; - clearSegment(cubeName); - - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); - long date4 = f.parse("2023-01-01").getTime(); - - return buildSegment(cubeName, date1, date4); - - } - - @SuppressWarnings("unused") - // called by reflection - private boolean testInnerJoinCubeWithView() throws Exception { + private boolean testLeftJoinCubeWithoutSlr() throws Exception { SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); List<String> result = Lists.newArrayList(); - final String cubeName = "test_kylin_cube_with_view_inner_join_empty"; + final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; clearSegment(cubeName); long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); + long date2 = f.parse("2012-06-01").getTime(); + long date3 = f.parse("2022-01-01").getTime(); long date4 = f.parse("2023-01-01").getTime(); - return buildSegment(cubeName, date1, date4); - - } - - @SuppressWarnings("unused") - // called by reflection - private boolean testLeftJoinCubeWithSlr() throws Exception { - String cubeName = "test_kylin_cube_with_slr_left_join_empty"; - clearSegment(cubeName); - - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); - long date2 = f.parse("2013-01-01").getTime(); - long date3 = f.parse("2013-07-01").getTime(); - long date4 = f.parse("2022-01-01").getTime(); - - List<String> result = Lists.newArrayList(); if (fastBuildMode) { return buildSegment(cubeName, date1, date4); } else { if (buildSegment(cubeName, date1, date2) == true) { if (buildSegment(cubeName, date2, date3) == true) { - if (buildSegment(cubeName, date3, date4) == true) { + if (buildSegment(cubeName, date3, date4) == true) { //empty segment return mergeSegment(cubeName, date1, date3);//don't merge all segments } } } } + return false; } @@ -413,10 +374,6 @@ public class BuildCubeWithEngine { DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST"); jobService.addJob(job); ExecutableState state = waitForJob(job.getId()); - // if (segment.getCubeDesc().getEngineType() == IEngineAware.ID_MR_V1 - // || segment.getCubeDesc().getStorageType() == IStorageAware.ID_SHARDED_HBASE) { - // checkHFilesInHBase(segment); - // } return Boolean.valueOf(ExecutableState.SUCCEED == state); }
