Repository: kylin Updated Branches: refs/heads/yang21 f56c3c40a -> b502a1741
KYLIN-2012 adjust rules for LookupTable & remove dead methods Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5565f77 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5565f77 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5565f77 Branch: refs/heads/yang21 Commit: d5565f775fa0e907a5a2c0bc33c75a2ece91ae31 Parents: f56c3c4 Author: gaodayue <gaoda...@meituan.com> Authored: Thu Oct 13 16:56:05 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Wed Oct 26 18:39:05 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/metadata/MetadataManager.java | 30 -------------------- .../apache/kylin/source/hive/SchemaChecker.java | 27 ++++++++++++++---- 2 files changed, 21 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d5565f77/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java index 8d77131..fd0a3c3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java @@ -461,11 +461,6 @@ public class MetadataManager { reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity)); } - public void reloadTableCache(String tableIdentity) throws IOException { - reloadSourceTableExt(tableIdentity); - reloadSourceTable(tableIdentity); - } - public DataModelDesc getDataModelDesc(String name) { return dataModelDescMap.get(name); } @@ -622,29 +617,4 @@ public class MetadataManager { srcTableExdMap.remove(tableIdentity); } - public String appendDBName(String table) { - - if (table.indexOf(".") > 0) - return table; - - Map<String, TableDesc> map = getAllTablesMap(); - - int count = 0; - String result = null; - for (TableDesc t : map.values()) { - if (t.getName().equalsIgnoreCase(table)) { - result = t.getIdentity(); - count++; - } - } - - if (count == 1) - return result; - - if (count > 1) { - logger.warn("There are more than 1 table named with '" + table + "' in different database; The program couldn't determine, randomly pick '" + result + "'"); - } - return result; - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/d5565f77/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java index 3b03551..0c5ccd0 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java @@ -134,7 +134,14 @@ public class SchemaChecker { } } - private List<String> checkAllUsedColumns(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) { + /** + * check whether all columns used in `cube` has compatible schema in current hive schema denoted by `fieldsMap`. + * @param cube cube to check, must use `table` in its model + * @param table kylin's table metadata + * @param fieldsMap current hive schema of `table` + * @return true if all columns used in `cube` has compatible schema with `fieldsMap`, false otherwise + */ + private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) { Set<ColumnDesc> usedColumns = Sets.newHashSet(); for (TblColRef col : cube.getAllColumns()) { usedColumns.add(col.getColumnDesc()); @@ -152,8 +159,15 @@ public class SchemaChecker { return violateColumns; } - private boolean checkAllColumns(TableDesc table, List<FieldSchema> fields) { - if (table.getColumnCount() != fields.size()) { + /** + * check whether all columns in `table` are still in `fields` and have the same index as before. + * + * @param table kylin's table metadata + * @param fields current table metadata in hive + * @return true if only new columns are appended in hive, false otherwise + */ + private boolean checkAllColumnsInTableDesc(TableDesc table, List<FieldSchema> fields) { + if (table.getColumnCount() > fields.size()) { return false; } @@ -194,15 +208,16 @@ public class SchemaChecker { // if user reloads a fact table used by cube, then all used columns // must match current schema if (factTable.getIdentity().equals(fullTableName)) { - List<String> violateColumns = checkAllUsedColumns(cube, factTable, currentFieldsMap); + List<String> violateColumns = checkAllColumnsInCube(cube, factTable, currentFieldsMap); if (!violateColumns.isEmpty()) { issues.add(format("Column %s used in cube[%s] and model[%s], but changed in hive", violateColumns, cube.getName(), modelName)); } } - // if user reloads a lookup table used by cube, then nearly all changes in schema are disallowed) + // if user reloads a lookup table used by cube, only append column(s) are allowed, all existing columns + // must be the same (except compatible type changes) for (TableDesc lookupTable : lookupTables) { - if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumns(lookupTable, currentFields)) { + if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumnsInTableDesc(lookupTable, currentFields)) { issues.add(format("Table '%s' is used as Lookup Table in cube[%s] and model[%s], but changed in hive", lookupTable.getIdentity(), cube.getName(), modelName)); } }
