KYLIN-1077 bug fix and add view to IT Signed-off-by: shaofengshi <shaofeng...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9284b47c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9284b47c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9284b47c Branch: refs/heads/master Commit: 9284b47c3545469344c8985e7d499238ee7b8a32 Parents: 7e4a59e Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Apr 29 11:36:19 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Apr 29 12:08:51 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../apache/kylin/dict/DictionaryManager.java | 3 +- .../apache/kylin/metadata/model/TableDesc.java | 16 +- .../cube/test_kylin_cube_with_view_empty.json | 10 ++ .../test_kylin_cube_with_view_desc.json | 169 +++++++++++++++++++ ...t_kylin_cube_without_slr_left_join_desc.json | 3 +- .../test_kylin_left_join_model_desc.json | 8 + .../localmeta/project/default.json | 6 + .../kylin/provision/BuildCubeWithEngine.java | 21 ++- .../apache/kylin/source/hive/HiveMRInput.java | 25 +-- 10 files changed, 234 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 8112ad5..4b63f38 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -206,8 +206,8 @@ public class CubeManager implements IRealizationProvider { TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable)); if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) { + String tableName = tableDesc.getMaterializedName(); tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable()); - String tableName = tableDesc.getHiveViewIntermediateTableName(); tableDesc.setName(tableName); } http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index bd9eac6..b94624c 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -28,7 +28,6 @@ import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.IRealizationSegment; import org.apache.kylin.source.ReadableTable; import org.apache.kylin.source.ReadableTable.TableSignature; import org.apache.kylin.source.SourceFactory; @@ -278,7 +277,7 @@ public class DictionaryManager { TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable)); if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) { tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable()); - String tableName = tableDesc.getHiveViewIntermediateTableName(); + String tableName = tableDesc.getMaterializedName(); tableDesc.setName(tableName); inpTable = SourceFactory.createReadableTable(tableDesc); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java index c028ac1..c5d2290 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java @@ -45,8 +45,8 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { private int sourceType = ISourceAware.ID_HIVE; @JsonProperty("table_type") private String tableType; - @JsonProperty("hive_view__table_name_prefix") - private String hiveViewIntermediateTableNamePrefix = "kylin_intermediate_"; + + private static final String materializedTableNamePrefix = "kylin_intermediate_"; private DatabaseDesc database = new DatabaseDesc(); @@ -59,6 +59,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { this.name = other.getName(); this.columns = other.getColumns(); this.database.setName(other.getDatabase()); + this.tableType = other.getTableType(); } public ColumnDesc findColumnByName(String name) { @@ -195,8 +196,8 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { return getIdentity().equals(other.getIdentity()); } - public String getHiveViewIntermediateTableName() { - return hiveViewIntermediateTableNamePrefix + "_" + database.getName() + "_" + name; + public String getMaterializedName() { + return materializedTableNamePrefix + database.getName() + "_" + name; } @Override @@ -228,11 +229,4 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { this.tableType = tableType; } - public String getHiveViewIntermediateTableNamePrefix() { - return hiveViewIntermediateTableNamePrefix; - } - - public void setHiveViewIntermediateTableNamePrefix(String hiveViewIntermediateTableNamePrefix) { - this.hiveViewIntermediateTableNamePrefix = hiveViewIntermediateTableNamePrefix; - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_empty.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_empty.json new file mode 100644 index 0000000..ed42df1 --- /dev/null +++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_empty.json @@ -0,0 +1,10 @@ +{ + "uuid" : "eeee3e80-41be-49a5-90ca-9fb7294db186", + "last_modified" : 0, + "name" : "test_kylin_cube_with_view_empty", + "owner" : null, + "descriptor" : "test_kylin_cube_with_view_desc", + "segments" : [ ], + "status" : "DISABLED", + "create_time" : null +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json new file mode 100644 index 0000000..ba0607a --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json @@ -0,0 +1,169 @@ +{ + "uuid" : "6789b7a8-3929-4dff-b59d-2100aadc8dbf", + "name" : "test_kylin_cube_with_view_desc", + "description" : null, + "dimensions" : [ { + "name" : "CAL_DT", + "table" : "EDW.V_TEST_CAL_DT", + "column" : "{FK}", + "derived" : [ "WEEK_BEG_DT" ] + }, { + "name" : "CATEGORY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "{FK}", + "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "META_CATEG_NAME", + "derived" : null + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL2_NAME", + "derived" : null + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL3_NAME", + "derived" : null + }, { + "name" : "LSTG_FORMAT_NAME", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "LSTG_FORMAT_NAME", + "derived" : null + }, { + "name" : "SITE_ID", + "table" : "EDW.TEST_SITES", + "column" : "{FK}", + "derived" : [ "SITE_NAME", "CRE_USER" ] + }, { + "name" : "SELLER_TYPE_CD", + "table" : "EDW.TEST_SELLER_TYPE_DIM", + "column" : "{FK}", + "derived" : [ "SELLER_TYPE_DESC" ] + } ], + "measures" : [ { + "name" : "GMV_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + }, + "dependent_measure_ref" : null + }, { + "name" : "GMV_MIN", + "function" : { + "expression" : "MIN", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + }, + "dependent_measure_ref" : null + }, { + "name" : "GMV_MAX", + "function" : { + "expression" : "MAX", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + }, + "dependent_measure_ref" : null + }, { + "name" : "TRANS_CNT", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "ITEM_COUNT_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "ITEM_COUNT", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "cal_dt", + "encoding" : "dict" + }, { + "column" : "leaf_categ_id", + "encoding" : "dict" + }, { + "column" : "meta_categ_name", + "encoding" : "dict" + }, { + "column" : "categ_lvl2_name", + "encoding" : "dict" + }, { + "column" : "categ_lvl3_name", + "encoding" : "dict" + }, { + "column" : "lstg_format_name", + "encoding" : "dict" + }, { + "column" : "lstg_site_id", + "encoding" : "dict" + }, { + "column" : "slr_segment_cd", + "encoding" : "dict" + } ] + }, + "signature" : null, + "last_modified" : 1448959801311, + "model_name" : "test_kylin_left_join_model_desc", + "null_string" : null, + "hbase_mapping" : { + "column_family" : [ { + "name" : "f1", + "columns" : [ { + "qualifier" : "m", + "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ] + } ] + }] + }, + "aggregation_groups" : [ { + "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"], + "select_rule" : { + "hierarchy_dims" : [ ], + "mandatory_dims" : [ "cal_dt" ], + "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ] + } + }, { + "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ], + "select_rule" : { + "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], + "mandatory_dims" : [ "cal_dt" ], + "joint_dims" : [ ] + } + } ], + "notify_list" : null, + "status_need_notify" : [ ], + "auto_merge_time_ranges" : null, + "retention_range" : 0, + "engine_type" : 2, + "storage_type" : 2, + "partition_date_start": 0 +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json index c2f2dd5..0ba85d9 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json @@ -1,11 +1,10 @@ { "uuid" : "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf", - "name" : "test_kylin_cube_without_slr_left_join_desc", "description" : null, "dimensions" : [ { "name" : "CAL_DT", - "table" : "EDW.V_TEST_CAL_DT", + "table" : "EDW.TEST_CAL_DT", "column" : "{FK}", "derived" : [ "WEEK_BEG_DT" ] }, { http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json index bc0eced..ddb4c81 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json @@ -4,6 +4,14 @@ "name": "test_kylin_left_join_model_desc", "lookups": [ { + "table": "EDW.TEST_CAL_DT", + "join": { + "type": "left", + "primary_key": ["CAL_DT"], + "foreign_key": ["CAL_DT"] + } + }, + { "table": "EDW.V_TEST_CAL_DT", "join": { "type": "left", http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/examples/test_case_data/localmeta/project/default.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index 41ac6da..bb195b6 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -42,11 +42,17 @@ "name": "test_kylin_hybrid_inner_join", "type": "HYBRID", "realization": "test_kylin_hybrid_inner_join" + }, + { + "name": "test_kylin_cube_with_view_empty", + "type": "CUBE", + "realization": "test_kylin_cube_with_view_empty" } ], "tables" : [ "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT", + "EDW.V_TEST_CAL_DT", "DEFAULT.TEST_CATEGORY_GROUPINGS", "EDW.TEST_SITES", "EDW.TEST_SELLER_TYPE_DIM", http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index 23696de..9343d29 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -181,7 +181,7 @@ public class BuildCubeWithEngine { } private void testLeft() throws Exception { - String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" }; + String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr", "testLeftJoinCubeWithView" }; runTestAndAssertSucceed(testCase); } @@ -311,6 +311,25 @@ public class BuildCubeWithEngine { } + + @SuppressWarnings("unused") + // called by reflection + private List<String> testLeftJoinCubeWithView() throws Exception { + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); + f.setTimeZone(TimeZone.getTimeZone("GMT")); + List<String> result = Lists.newArrayList(); + final String cubeName = "test_kylin_cube_with_view_empty"; + clearSegment(cubeName); + + long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); + long date4 = f.parse("2023-01-01").getTime(); + + result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date1, date4)); + + return result; + + } + @SuppressWarnings("unused") // called by reflection private List<String> testLeftJoinCubeWithSlr() throws Exception { http://git-wip-us.apache.org/repos/asf/kylin/blob/9284b47c/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 7e5ed0b..b2ce600 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -155,15 +155,15 @@ public class HiveMRInput implements IMRInput { for(TableDesc lookUpTableDesc : cubeDesc.getLookupTableDescs()) { if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(lookUpTableDesc.getTableType())) { findHiveViewLookUpTable = true; - lookUpTableDesc.setHiveViewIntermediateTableNamePrefix("kylin_intermediate_" + jobId); StringBuilder createIntermediateTableHql = new StringBuilder(); + createIntermediateTableHql.append("DROP TABLE IF EXISTS " + lookUpTableDesc.getMaterializedName() + ";\n"); createIntermediateTableHql.append("CREATE TABLE IF NOT EXISTS " + - lookUpTableDesc.getHiveViewIntermediateTableName() + "\n"); + lookUpTableDesc.getMaterializedName() + "\n"); createIntermediateTableHql.append("LOCATION '" + JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/" + - lookUpTableDesc.getHiveViewIntermediateTableName() + "'\n"); + lookUpTableDesc.getMaterializedName() + "'\n"); createIntermediateTableHql.append("AS SELECT * FROM " + lookUpTableDesc.getIdentity() + ";\n"); hiveCmdBuilder.addStatement(createIntermediateTableHql.toString()); - hiveViewIntermediateTables = hiveViewIntermediateTables + lookUpTableDesc.getHiveViewIntermediateTableName() + ";"; + hiveViewIntermediateTables = hiveViewIntermediateTables + lookUpTableDesc.getMaterializedName() + ";"; } if (findHiveViewLookUpTable) { hiveViewIntermediateTables= hiveViewIntermediateTables.substring(0, hiveViewIntermediateTables.length()-1); @@ -184,7 +184,7 @@ public class HiveMRInput implements IMRInput { step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION); step.setIntermediateTableIdentity(getIntermediateTableIdentity()); step.setExternalDataPath(JoinedFlatTable.getTableDir(flatHiveTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId()))); - step.setHiveViewIntermediateTableIdentitys(hiveViewIntermediateTables); + step.setHiveViewIntermediateTableIdentities(hiveViewIntermediateTables); jobFlow.addTask(step); } @@ -205,7 +205,8 @@ public class HiveMRInput implements IMRInput { StringBuffer output = new StringBuffer(); try { output.append(cleanUpIntermediateFlatTable(config)); - output.append(cleanUpHiveViewIntermediateTable(config)); + // don't drop view to avoid concurrent issue + //output.append(cleanUpHiveViewIntermediateTable(config)); } catch (IOException e) { logger.error("job:" + getId() + " execute finished with exception", e); return new ExecuteResult(ExecuteResult.State.ERROR, e.getMessage()); @@ -251,13 +252,13 @@ public class HiveMRInput implements IMRInput { StringBuffer output = new StringBuffer(); final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatement("USE " + config.getHiveDatabaseForIntermediateTable() + ";"); - if (!getHiveViewIntermediateTableIdentitys().isEmpty()) { - for(String hiveTableName : getHiveViewIntermediateTableIdentitys().split(";")) { + if (getHiveViewIntermediateTableIdentities() != null && !getHiveViewIntermediateTableIdentities().isEmpty()) { + for(String hiveTableName : getHiveViewIntermediateTableIdentities().split(";")) { hiveCmdBuilder.addStatement("DROP TABLE IF EXISTS " + hiveTableName + ";"); } } config.getCliCommandExecutor().execute(hiveCmdBuilder.build()); - output.append("hive view intermediate tables: " + getHiveViewIntermediateTableIdentitys() + " is dropped. \n"); + output.append("hive view intermediate tables: " + getHiveViewIntermediateTableIdentities() + " is dropped. \n"); return output.toString(); } @@ -277,11 +278,11 @@ public class HiveMRInput implements IMRInput { return getParam("externalDataPath"); } - public void setHiveViewIntermediateTableIdentitys(String tableIdentitys) { - setParam("oldHiveViewIntermediateTables", tableIdentitys); + public void setHiveViewIntermediateTableIdentities(String tableIdentities) { + setParam("oldHiveViewIntermediateTables", tableIdentities); } - private String getHiveViewIntermediateTableIdentitys() { + private String getHiveViewIntermediateTableIdentities() { return getParam("oldHiveViewIntermediateTables"); } }
