KYLIN-2291 Collect hive table property skip-header-line-count Signed-off-by: Li Yang <liy...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d79d9374 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d79d9374 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d79d9374 Branch: refs/heads/KYLIN-2131 Commit: d79d9374cd8411518ee8eafc7640910826f6838c Parents: f2377db Author: Cheng Wang <cheng.w...@kyligence.io> Authored: Fri Dec 16 17:23:15 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri Dec 16 17:45:16 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/source/hive/BeelineHiveClient.java | 3 +++ .../java/org/apache/kylin/source/hive/CLIHiveClient.java | 2 +- .../apache/kylin/source/hive/HiveSourceTableLoader.java | 1 + .../java/org/apache/kylin/source/hive/HiveTableMeta.java | 4 +++- .../apache/kylin/source/hive/HiveTableMetaBuilder.java | 11 ++++++++++- 5 files changed, 18 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java index 47b551b..468ccb1 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java @@ -193,6 +193,9 @@ public class BeelineHiveClient implements IHiveClient { if ("numFiles".equals(resultSet.getString(2).trim())) { builder.setFileNum(Long.parseLong(resultSet.getString(3).trim())); } + if ("skip.header.line.count".equals(resultSet.getString(2).trim())) { + builder.setSkipHeaderLineCount(resultSet.getString(3).trim()); + } } } if ("InputFormat:".equals(resultSet.getString(1).trim())) { http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java index 5a17f1f..e8a93bd 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java @@ -42,7 +42,6 @@ import com.google.common.collect.Lists; * */ public class CLIHiveClient implements IHiveClient { - protected HiveConf hiveConf = null; protected Driver driver = null; protected HiveMetaStoreClient metaStoreClient = null; @@ -109,6 +108,7 @@ public class CLIHiveClient implements IHiveClient { builder.setOwner(table.getOwner()); builder.setLastAccessTime(table.getLastAccessTime()); builder.setTableType(table.getTableType()); + builder.setSkipHeaderLineCount(table.getParameters().get("skip.header.line.count")); return builder.createHiveTableMeta(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java index 57292dc..77e1084 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java @@ -140,6 +140,7 @@ public class HiveSourceTableLoader { tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(hiveTableMeta.fileNum)); tableExtDesc.addDataSourceProp("hive_inputFormat", hiveTableMeta.sdInputFormat); tableExtDesc.addDataSourceProp("hive_outputFormat", hiveTableMeta.sdOutputFormat); + tableExtDesc.addDataSourceProp("skip_header_line_count", String.valueOf(hiveTableMeta.skipHeaderLineCount)); metaMgr.saveTableExt(tableExtDesc); metaMgr.saveSourceTable(tableDesc); http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java index 784a0bb..fa9eb29 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java @@ -44,6 +44,7 @@ class HiveTableMeta { String sdOutputFormat; String owner; String tableType; + int skipHeaderLineCount; int lastAccessTime; long fileSize; long fileNum; @@ -51,7 +52,7 @@ class HiveTableMeta { List<HiveTableColumnMeta> allColumns; List<HiveTableColumnMeta> partitionColumns; - public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta> partitionColumns) { + public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum, int skipHeaderLineCount, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta> partitionColumns) { this.tableName = tableName; this.sdLocation = sdLocation; this.sdInputFormat = sdInputFormat; @@ -64,6 +65,7 @@ class HiveTableMeta { this.isNative = isNative; this.allColumns = allColumns; this.partitionColumns = partitionColumns; + this.skipHeaderLineCount = skipHeaderLineCount; } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java index 7a3e5d6..073ded5 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java @@ -32,6 +32,7 @@ public class HiveTableMetaBuilder { private int lastAccessTime; private long fileSize; private long fileNum; + private int skipHeaderLineCount; private boolean isNative = true; private List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList(); private List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList(); @@ -81,6 +82,14 @@ public class HiveTableMetaBuilder { return this; } + public HiveTableMetaBuilder setSkipHeaderLineCount(String skipHeaderLineCount) { + if (null == skipHeaderLineCount) + this.skipHeaderLineCount = 0; + else + this.skipHeaderLineCount = Integer.parseInt(skipHeaderLineCount); + return this; + } + public HiveTableMetaBuilder setIsNative(boolean isNative) { this.isNative = isNative; return this; @@ -97,6 +106,6 @@ public class HiveTableMetaBuilder { } public HiveTableMeta createHiveTableMeta() { - return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner, tableType, lastAccessTime, fileSize, fileNum, isNative, allColumns, partitionColumns); + return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner, tableType, lastAccessTime, fileSize, fileNum, skipHeaderLineCount, isNative, allColumns, partitionColumns); } } \ No newline at end of file
