KYLIN-2323 refactor Table Ext Info, display table type
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b052d6f2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b052d6f2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b052d6f2 Branch: refs/heads/sparkcubing-rebase Commit: b052d6f22930c45bd34f1d7a2fd493cbf4330791 Parents: e7e29f1 Author: Billy Liu <billy...@apache.org> Authored: Tue Jan 3 11:15:06 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Tue Jan 3 11:15:06 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/metadata/MetadataManager.java | 4 +- .../kylin/metadata/model/TableExtDesc.java | 52 +------------------- .../kylin/rest/response/TableDescResponse.java | 1 + .../apache/kylin/rest/service/TableService.java | 21 ++++---- .../source/hive/HiveSourceTableLoader.java | 16 ++---- webapp/app/js/services/tables.js | 2 - webapp/app/partials/tables/table_detail.html | 16 +++--- 7 files changed, 28 insertions(+), 84 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java index 3b6eb61..49ec96e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java @@ -504,7 +504,7 @@ public class MetadataManager { List<String> models = new ArrayList<>(); for (DataModelDesc modelDesc : getModels(projectName)) { for (TableRef tableRef : modelDesc.getAllTables()) { - if (tableRef.getTableName().equalsIgnoreCase(tableName)) { + if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) { models.add(modelDesc.getName()); } } @@ -515,7 +515,7 @@ public class MetadataManager { public boolean isTableInAnyModel(String tableName) { for (DataModelDesc modelDesc : getModels()) { for (TableRef tableRef : modelDesc.getAllTables()) { - if (tableRef.getTableName().equalsIgnoreCase(tableName)) { + if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) { return true; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java index 6393dbf..97dd2c1 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java @@ -47,18 +47,8 @@ public class TableExtDesc extends RootPersistentEntity { @JsonProperty("sample_rows") private List<String[]> sampleRows = new ArrayList<>(); - @JsonProperty("storage_location") - private String storageLocation; - @JsonProperty("owner") - private String owner; - @JsonProperty("last_access_time") - private String lastAccessTime; @JsonProperty("last_modified_time") private String lastModifiedTime; - @JsonProperty("partition_column") - private String partitionColumn; - @JsonProperty("total_file_size") - private String totalFileSize; @JsonProperty("total_rows") private String totalRows; @JsonProperty("data_source_properties") @@ -161,22 +151,6 @@ public class TableExtDesc extends RootPersistentEntity { this.tableName = this.tableName.toUpperCase(); } - public void setStorageLocation(String storageLocation) { - this.storageLocation = storageLocation; - } - - public String getStorageLocation() { - return this.storageLocation; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getOwner() { - return this.owner; - } - public void setLastModifiedTime(String lastModifiedTime) { this.lastModifiedTime = lastModifiedTime; } @@ -185,32 +159,8 @@ public class TableExtDesc extends RootPersistentEntity { return this.lastModifiedTime; } - public void setLastAccessTime(String lastAccessTime) { - this.lastAccessTime = lastAccessTime; - } - - public String getLastAccessTime() { - return this.lastAccessTime; - } - - public void setPartitionColumn(String partitionColumn) { - this.partitionColumn = partitionColumn; - } - - public String getPartitionColumn() { - return this.partitionColumn; - } - public boolean isPartitioned() { - return this.partitionColumn == null ? false : !this.partitionColumn.isEmpty(); - } - - public void setTotalFileSize(String totalFileSize) { - this.totalFileSize = totalFileSize; - } - - public String getTotalFileSize() { - return this.totalFileSize; + return this.dataSourceProps.get("partition_column") == null ? false : !this.dataSourceProps.get("partition_column").isEmpty(); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java index f53b951..9bc5360 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java +++ b/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java @@ -76,6 +76,7 @@ public class TableDescResponse extends TableDesc { this.setName(table.getName()); this.setSourceType(table.getSourceType()); this.setUuid(table.getUuid()); + this.setTableType(table.getTableType()); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index 3325907..e3df60a 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.UUID; import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.engine.mr.HadoopUtil; import org.apache.kylin.engine.mr.common.HadoopShellExecutable; import org.apache.kylin.engine.mr.common.MapReduceExecutable; @@ -91,7 +92,7 @@ public class TableService extends BasicService { return table; } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN) + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public String[] loadHiveTablesToProject(String[] tables, String project) throws IOException { Set<String> loaded = HiveSourceTableLoader.loadHiveTables(tables, getConfig()); String[] result = (String[]) loaded.toArray(new String[loaded.size()]); @@ -99,18 +100,17 @@ public class TableService extends BasicService { return result; } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) private void unLoadHiveTable(String tableName) throws IOException { tableName = normalizeHiveTableName(tableName); - HiveSourceTableLoader.unLoadHiveTable(tableName); + MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + metaMgr.removeSourceTable(tableName); + metaMgr.removeTableExt(tableName); } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) private void syncTableToProject(String[] tables, String project) throws IOException { getProjectManager().addTableDescToProject(tables, project); } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) private void removeTableFromProject(String tableName, String projectName) throws IOException { tableName = normalizeHiveTableName(tableName); getProjectManager().removeTableDescFromProject(tableName, projectName); @@ -123,6 +123,7 @@ public class TableService extends BasicService { * @param project * @return */ + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public boolean unLoadHiveTable(String tableName, String project) { boolean rtn = false; int tableType = 0; @@ -178,6 +179,7 @@ public class TableService extends BasicService { * @param project * @throws IOException */ + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public void addStreamingTable(TableDesc desc, String project) throws IOException { desc.setUuid(UUID.randomUUID().toString()); getMetadataManager().saveSourceTable(desc); @@ -230,11 +232,6 @@ public class TableService extends BasicService { rtableDesc.setCardinality(cardinality); } dataSourceProp.putAll(tableExtDesc.getDataSourceProp()); - dataSourceProp.put("location", tableExtDesc.getStorageLocation()); - dataSourceProp.put("owner", tableExtDesc.getOwner()); - dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime()); - dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn()); - dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize()); rtableDesc.setDescExd(dataSourceProp); return rtableDesc; } @@ -277,8 +274,8 @@ public class TableService extends BasicService { TableDesc table = getMetadataManager().getTableDesc(tableName); final TableExtDesc tableExt = getMetadataManager().getTableExt(tableName); if (table == null) { - IllegalArgumentException e = new IllegalArgumentException("Cannot find table descirptor " + tableName); - logger.error("Cannot find table descirptor " + tableName, e); + IllegalArgumentException e = new IllegalArgumentException("Cannot find table descriptor " + tableName); + logger.error("Cannot find table descriptor " + tableName, e); throw e; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java index b56009a..286ffac 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java @@ -76,12 +76,6 @@ public class HiveSourceTableLoader { return loadedTables; } - public static void unLoadHiveTable(String hiveTable) throws IOException { - MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - metaMgr.removeSourceTable(hiveTable); - metaMgr.removeTableExt(hiveTable); - } - private static List<String> extractHiveTables(String database, Set<String> tables, IHiveClient hiveClient) throws IOException { List<String> loadedTables = Lists.newArrayList(); @@ -132,11 +126,11 @@ public class HiveSourceTableLoader { } TableExtDesc tableExtDesc = metaMgr.getTableExt(tableDesc.getIdentity()); - tableExtDesc.setStorageLocation(hiveTableMeta.sdLocation); - tableExtDesc.setOwner(hiveTableMeta.owner); - tableExtDesc.setLastAccessTime(String.valueOf(hiveTableMeta.lastAccessTime)); - tableExtDesc.setPartitionColumn(partitionColumnString.toString()); - tableExtDesc.setTotalFileSize(String.valueOf(hiveTableMeta.fileSize)); + tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation); + tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner); + tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime)); + tableExtDesc.addDataSourceProp("partition_column", partitionColumnString.toString()); + tableExtDesc.addDataSourceProp("total_file_size", String.valueOf(hiveTableMeta.fileSize)); tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(hiveTableMeta.fileNum)); tableExtDesc.addDataSourceProp("hive_inputFormat", hiveTableMeta.sdInputFormat); tableExtDesc.addDataSourceProp("hive_outputFormat", hiveTableMeta.sdOutputFormat); http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/webapp/app/js/services/tables.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js index 4e7a7c4..38bf8a0 100755 --- a/webapp/app/js/services/tables.js +++ b/webapp/app/js/services/tables.js @@ -20,8 +20,6 @@ KylinApp.factory('TableService', ['$resource', function ($resource, config) { return $resource(Config.service.url + 'tables/:tableName/:action/:database', {}, { list: {method: 'GET', params: {}, cache: true, isArray: true}, get: {method: 'GET', params: {}, isArray: false}, - getExd: {method: 'GET', params: {action: 'exd-map'}, isArray: false}, - reload: {method: 'PUT', params: {action: 'reload'}, isArray: false}, loadHiveTable: {method: 'POST', params: {}, isArray: false}, unLoadHiveTable: {method: 'DELETE', params: {}, isArray: false}, genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false}, http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/webapp/app/partials/tables/table_detail.html ---------------------------------------------------------------------- diff --git a/webapp/app/partials/tables/table_detail.html b/webapp/app/partials/tables/table_detail.html index 6399422..f227826 100644 --- a/webapp/app/partials/tables/table_detail.html +++ b/webapp/app/partials/tables/table_detail.html @@ -39,7 +39,7 @@ <table class="table"> <tbody> <tr> - <th style="width:20%">NAME</th> + <th style="width:20%">TABLE NAME</th> <td>{{ tableModel.selectedSrcTable.name}}</td> </tr> <tr> @@ -47,6 +47,10 @@ <td>{{tableModel.selectedSrcTable.database}}</td> </tr> <tr> + <th>TABLE TYPE</th> + <td>{{tableModel.selectedSrcTable.table_type}}</td> + </tr> + <tr> <th>SNAPSHOT TIME</th> <td>{{tableModel.selectedSrcTable.exd.lastUpdateTime | utcToConfigTimeZone}}</td> </tr> @@ -56,11 +60,11 @@ </tr> <tr> <th>INPUT FORMAT</th> - <td>{{tableModel.selectedSrcTable.exd.inputformat}}</td> + <td>{{tableModel.selectedSrcTable.exd.hive_inputFormat}}</td> </tr> <tr> <th>OUTPUT FORMAT</th> - <td>{{tableModel.selectedSrcTable.exd.outputformat}}</td> + <td>{{tableModel.selectedSrcTable.exd.hive_outputFormat}}</td> </tr> <tr> <th>OWNER</th> @@ -70,11 +74,11 @@ </tr> <tr> <th>TOTAL FILE NUMBER</th> - <td>{{tableModel.selectedSrcTable.exd.totalNumberFiles}}</td> + <td>{{tableModel.selectedSrcTable.exd.total_file_number}}</td> </tr> <tr> <th>TOTAL FILE SIZE</th> - <td>{{tableModel.selectedSrcTable.exd.totalFileSize}}</td> + <td>{{tableModel.selectedSrcTable.exd.total_file_size}}</td> </tr> <tr> <th>PARTITIONED</th> @@ -82,7 +86,7 @@ </tr> <tr> <th>PARTITION COLUMNS</th> - <td>{{tableModel.selectedSrcTable.exd.partitionColumns}}</td> + <td>{{tableModel.selectedSrcTable.exd.partition_column}}</td> </tr> </tbody> </table>
