KYLIN-1789 Couldn't use View as Lookup when join type is "inner" Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/930f397d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/930f397d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/930f397d
Branch: refs/heads/stream_m1 Commit: 930f397d8e763e91de4741b3b5dc1096668f3fc6 Parents: a179136 Author: shaofengshi <shaofeng...@apache.org> Authored: Thu Jun 16 13:54:48 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Thu Jun 16 13:56:21 2016 +0800 ---------------------------------------------------------------------- ...t_kylin_cube_with_view_inner_join_empty.json | 10 ++ .../test_kylin_cube_with_view_desc.json | 2 +- ...st_kylin_cube_with_view_inner_join_desc.json | 169 +++++++++++++++++++ .../test_kylin_inner_join_view_model_desc.json | 115 +++++++++++++ .../test_kylin_left_join_model_desc.json | 15 -- .../test_kylin_left_join_view_model_desc.json | 115 +++++++++++++ .../localmeta/project/default.json | 17 +- .../kylin/provision/BuildCubeWithEngine.java | 20 ++- .../apache/kylin/source/hive/HiveMRInput.java | 13 +- 9 files changed, 448 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json new file mode 100644 index 0000000..6215174 --- /dev/null +++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json @@ -0,0 +1,10 @@ +{ + "uuid" : "ffff3e80-41be-49a5-90ca-9fb7294db186", + "last_modified" : 0, + "name" : "test_kylin_cube_with_view_inner_join_empty", + "owner" : null, + "descriptor" : "test_kylin_cube_with_view_inner_join_desc", + "segments" : [ ], + "status" : "DISABLED", + "create_time" : null +} http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json index ba0607a..e66dc24 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json @@ -133,7 +133,7 @@ }, "signature" : null, "last_modified" : 1448959801311, - "model_name" : "test_kylin_left_join_model_desc", + "model_name" : "test_kylin_left_join_view_model_desc", "null_string" : null, "hbase_mapping" : { "column_family" : [ { http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json new file mode 100644 index 0000000..d4c64b5 --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json @@ -0,0 +1,169 @@ +{ + "uuid" : "9876b7a8-3929-4dff-b59d-2100aadc8dbf", + "name" : "test_kylin_cube_with_view_inner_join_desc", + "description" : null, + "dimensions" : [ { + "name" : "CAL_DT", + "table" : "EDW.V_TEST_CAL_DT", + "column" : "{FK}", + "derived" : [ "WEEK_BEG_DT" ] + }, { + "name" : "CATEGORY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "{FK}", + "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "META_CATEG_NAME", + "derived" : null + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL2_NAME", + "derived" : null + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL3_NAME", + "derived" : null + }, { + "name" : "LSTG_FORMAT_NAME", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "LSTG_FORMAT_NAME", + "derived" : null + }, { + "name" : "SITE_ID", + "table" : "EDW.TEST_SITES", + "column" : "{FK}", + "derived" : [ "SITE_NAME", "CRE_USER" ] + }, { + "name" : "SELLER_TYPE_CD", + "table" : "EDW.TEST_SELLER_TYPE_DIM", + "column" : "{FK}", + "derived" : [ "SELLER_TYPE_DESC" ] + } ], + "measures" : [ { + "name" : "GMV_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + }, + "dependent_measure_ref" : null + }, { + "name" : "GMV_MIN", + "function" : { + "expression" : "MIN", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + }, + "dependent_measure_ref" : null + }, { + "name" : "GMV_MAX", + "function" : { + "expression" : "MAX", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + }, + "dependent_measure_ref" : null + }, { + "name" : "TRANS_CNT", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "ITEM_COUNT_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "ITEM_COUNT", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "cal_dt", + "encoding" : "dict" + }, { + "column" : "leaf_categ_id", + "encoding" : "dict" + }, { + "column" : "meta_categ_name", + "encoding" : "dict" + }, { + "column" : "categ_lvl2_name", + "encoding" : "dict" + }, { + "column" : "categ_lvl3_name", + "encoding" : "dict" + }, { + "column" : "lstg_format_name", + "encoding" : "dict" + }, { + "column" : "lstg_site_id", + "encoding" : "dict" + }, { + "column" : "slr_segment_cd", + "encoding" : "dict" + } ] + }, + "signature" : null, + "last_modified" : 1448959801311, + "model_name" : "test_kylin_inner_join_view_model_desc", + "null_string" : null, + "hbase_mapping" : { + "column_family" : [ { + "name" : "f1", + "columns" : [ { + "qualifier" : "m", + "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ] + } ] + }] + }, + "aggregation_groups" : [ { + "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"], + "select_rule" : { + "hierarchy_dims" : [ ], + "mandatory_dims" : [ "cal_dt" ], + "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ] + } + }, { + "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ], + "select_rule" : { + "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], + "mandatory_dims" : [ "cal_dt" ], + "joint_dims" : [ ] + } + } ], + "notify_list" : null, + "status_need_notify" : [ ], + "auto_merge_time_ranges" : null, + "retention_range" : 0, + "engine_type" : 2, + "storage_type" : 2, + "partition_date_start": 0 +} http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json new file mode 100644 index 0000000..54e9ffb --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json @@ -0,0 +1,115 @@ +{ + + "uuid": "8b184ee2-1ccb-4b07-a38e-4c298563e0f7", + "name": "test_kylin_inner_join_view_model_desc", + "lookups": [ + { + "table": "EDW.TEST_CAL_DT", + "join": { + "type": "inner", + "primary_key": ["CAL_DT"], + "foreign_key": ["CAL_DT"] + } + }, + { + "table": "EDW.V_TEST_CAL_DT", + "join": { + "type": "inner", + "primary_key": ["CAL_DT"], + "foreign_key": ["CAL_DT"] + } + }, + { + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "join": { + "type": "inner", + "primary_key": [ + "LEAF_CATEG_ID", + "SITE_ID" + ], + "foreign_key": [ + "LEAF_CATEG_ID", + "LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SITES", + "join": { + "type": "inner", + "primary_key": ["SITE_ID"], + "foreign_key": ["LSTG_SITE_ID"] + } + }, + { + "table": "EDW.TEST_SELLER_TYPE_DIM", + "join": { + "type": "inner", + "primary_key": ["SELLER_TYPE_CD"], + "foreign_key": ["SLR_SEGMENT_CD"] + } + } + ], + "dimensions": [ + { + "table": "default.test_kylin_fact", + "columns": [ + "TRANS_ID", + "CAL_DT", + "lstg_format_name", + "LSTG_SITE_ID", + "SLR_SEGMENT_CD", + "SELLER_ID" + ] + }, + { + "table": "default.test_category_groupings", + "columns": [ + "leaf_categ_id", + "site_id", + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER", + "meta_categ_name", + "categ_lvl2_name", + "categ_lvl3_name" + ] + }, + { + "table": "edw.test_sites", + "columns": [ + "site_id", + "site_name", + "cre_user" + ] + }, + { + "table": "edw.test_seller_type_dim", + "columns": [ + "seller_type_cd", + "seller_type_desc" + ] + }, + { + "table": "edw.v_test_cal_dt", + "columns": [ + "cal_dt", + "week_beg_dt" + ] + } + ], + "metrics": [ + "PRICE", + "ITEM_COUNT", + "SELLER_ID" + ], + "last_modified": 1422435345352, + "fact_table": "DEFAULT.TEST_KYLIN_FACT", + "filter_condition": null, + "partition_desc": { + "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt", + "partition_date_start": 0, + "partition_type": "APPEND" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json index 8be6dc5..bedb167 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json @@ -12,14 +12,6 @@ } }, { - "table": "EDW.V_TEST_CAL_DT", - "join": { - "type": "left", - "primary_key": ["CAL_DT"], - "foreign_key": ["CAL_DT"] - } - }, - { "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", "join": { "type": "left", @@ -90,13 +82,6 @@ "seller_type_cd", "seller_type_desc" ] - }, - { - "table": "edw.v_test_cal_dt", - "columns": [ - "cal_dt", - "week_beg_dt" - ] } ], "metrics": [ http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json new file mode 100644 index 0000000..7c009a1 --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json @@ -0,0 +1,115 @@ +{ + + "uuid": "9c184ee2-1ccb-4b07-a38e-4c298563e0f7", + "name": "test_kylin_left_join_view_model_desc", + "lookups": [ + { + "table": "EDW.TEST_CAL_DT", + "join": { + "type": "left", + "primary_key": ["CAL_DT"], + "foreign_key": ["CAL_DT"] + } + }, + { + "table": "EDW.V_TEST_CAL_DT", + "join": { + "type": "left", + "primary_key": ["CAL_DT"], + "foreign_key": ["CAL_DT"] + } + }, + { + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "join": { + "type": "left", + "primary_key": [ + "LEAF_CATEG_ID", + "SITE_ID" + ], + "foreign_key": [ + "LEAF_CATEG_ID", + "LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SITES", + "join": { + "type": "left", + "primary_key": ["SITE_ID"], + "foreign_key": ["LSTG_SITE_ID"] + } + }, + { + "table": "EDW.TEST_SELLER_TYPE_DIM", + "join": { + "type": "left", + "primary_key": ["SELLER_TYPE_CD"], + "foreign_key": ["SLR_SEGMENT_CD"] + } + } + ], + "dimensions": [ + { + "table": "default.test_kylin_fact", + "columns": [ + "TRANS_ID", + "CAL_DT", + "lstg_format_name", + "LSTG_SITE_ID", + "SLR_SEGMENT_CD", + "SELLER_ID" + ] + }, + { + "table": "default.test_category_groupings", + "columns": [ + "leaf_categ_id", + "site_id", + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER", + "meta_categ_name", + "categ_lvl2_name", + "categ_lvl3_name" + ] + }, + { + "table": "edw.test_sites", + "columns": [ + "site_id", + "site_name", + "cre_user" + ] + }, + { + "table": "edw.test_seller_type_dim", + "columns": [ + "seller_type_cd", + "seller_type_desc" + ] + }, + { + "table": "edw.v_test_cal_dt", + "columns": [ + "cal_dt", + "week_beg_dt" + ] + } + ], + "metrics": [ + "PRICE", + "ITEM_COUNT", + "SELLER_ID" + ], + "last_modified": 1422435345352, + "fact_table": "DEFAULT.TEST_KYLIN_FACT", + "filter_condition": null, + "partition_desc": { + "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt", + "partition_date_start": 0, + "partition_type": "APPEND" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/examples/test_case_data/localmeta/project/default.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index bb195b6..127d57f 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -42,17 +42,22 @@ "name": "test_kylin_hybrid_inner_join", "type": "HYBRID", "realization": "test_kylin_hybrid_inner_join" - }, - { - "name": "test_kylin_cube_with_view_empty", - "type": "CUBE", - "realization": "test_kylin_cube_with_view_empty" + }, + { + "name": "test_kylin_cube_with_view_empty", + "type": "CUBE", + "realization": "test_kylin_cube_with_view_empty" + }, + { + "name": "test_kylin_cube_with_view_inner_join_empty", + "type": "CUBE", + "realization": "test_kylin_cube_with_view_inner_join_empty" } ], "tables" : [ "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT", - "EDW.V_TEST_CAL_DT", + "EDW.V_TEST_CAL_DT", "DEFAULT.TEST_CATEGORY_GROUPINGS", "EDW.TEST_SITES", "EDW.TEST_SELLER_TYPE_DIM", http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index 063f0ed..6a0a351 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -190,7 +190,7 @@ public class BuildCubeWithEngine { } private void testViewAsLookup() throws Exception { - String[] testCase = new String[] { "testLeftJoinCubeWithView" }; + String[] testCase = new String[] { "testInnerJoinCubeWithView", "testLeftJoinCubeWithView" }; runTestAndAssertSucceed(testCase); } @@ -344,6 +344,24 @@ public class BuildCubeWithEngine { @SuppressWarnings("unused") // called by reflection + private List<String> testInnerJoinCubeWithView() throws Exception { + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); + f.setTimeZone(TimeZone.getTimeZone("GMT")); + List<String> result = Lists.newArrayList(); + final String cubeName = "test_kylin_cube_with_view_inner_join_empty"; + clearSegment(cubeName); + + long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); + long date4 = f.parse("2023-01-01").getTime(); + + result.add(buildSegment(cubeName, date1, date4)); + + return result; + + } + + @SuppressWarnings("unused") + // called by reflection private List<String> testLeftJoinCubeWithSlr() throws Exception { String cubeName = "test_kylin_cube_with_slr_left_join_empty"; clearSegment(cubeName); http://git-wip-us.apache.org/repos/asf/kylin/blob/930f397d/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index a444903..5af8eb9 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -45,7 +45,9 @@ import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecuteResult; +import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; +import org.apache.kylin.metadata.model.LookupDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.realization.IRealizationSegment; @@ -144,15 +146,16 @@ public class HiveMRInput implements IMRInput { CubeManager cubeMgr = CubeManager.getInstance(kylinConfig); String cubeName = seg.getRealization().getName(); CubeDesc cubeDesc = cubeMgr.getCube(cubeName).getDescriptor(); - + MetadataManager metadataManager = MetadataManager.getInstance(kylinConfig); final Set<TableDesc> lookupViewsTables = Sets.newHashSet(); - for(DimensionDesc dimensionDesc : cubeDesc.getDimensions()) { - TableDesc tableDesc = dimensionDesc.getTableDesc(); - if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType()) - && !cubeDesc.getModel().getFactTableDesc().equals(tableDesc)) { + + for (LookupDesc lookupDesc : cubeDesc.getModel().getLookups()) { + TableDesc tableDesc = metadataManager.getTableDesc(lookupDesc.getTable()); + if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) { lookupViewsTables.add(tableDesc); } } + if(lookupViewsTables.size() == 0) { return null; }
