KYLIN-3167, fulfil datatype's precision
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55365b68 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55365b68 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55365b68 Branch: refs/heads/sync Commit: 55365b68dfa021f5c14acbd79e2417f55b294473 Parents: 0988c84 Author: Cheng Wang <cheng.w...@kyligence.io> Authored: Tue Jan 16 14:55:36 2018 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri Jan 26 22:54:58 2018 +0800 ---------------------------------------------------------------------- .../kylin/source/hive/BeelineHiveClient.java | 27 +++++++++-- .../source/hive/BeelineHIveClientTest.java | 51 ++++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/55365b68/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java index 314402c..747b1bb 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java @@ -114,7 +114,7 @@ public class BeelineHiveClient implements IHiveClient { } return count; } - + @Override public void executeHQL(String hql) throws IOException { throw new UnsupportedOperationException(); @@ -132,7 +132,11 @@ public class BeelineHiveClient implements IHiveClient { List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList(); while (columns.next()) { - allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), columns.getString(12))); + String columnName = columns.getString(4); + String dataType = columns.getString(6); + String comment = columns.getString(12); + dataType = considerDataTypePrecision(dataType, columns.getString(7), columns.getString(9)); + allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columnName, dataType, comment)); } builder.setAllColumns(allColumns); DBUtils.closeQuietly(columns); @@ -143,6 +147,19 @@ public class BeelineHiveClient implements IHiveClient { return builder.createHiveTableMeta(); } + public static String considerDataTypePrecision(String dataType, String precision, String scale) { + if ("VARCHAR".equalsIgnoreCase(dataType) || "CHAR".equalsIgnoreCase(dataType)) { + if (null != precision) + dataType = new StringBuilder(dataType).append("(").append(precision).append(")").toString(); + } + if ("DECIMAL".equalsIgnoreCase(dataType) || "NUMERIC".equalsIgnoreCase(dataType)) { + if (precision != null && scale != null) + dataType = new StringBuilder(dataType).append("(").append(precision).append(",").append(scale) + .append(")").toString(); + } + return dataType; + } + private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException { while (resultSet.next()) { @@ -156,7 +173,8 @@ public class BeelineHiveClient implements IHiveClient { if ("".equals(resultSet.getString(1).trim())) { break; } - partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim(), resultSet.getString(3).trim())); + partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), + resultSet.getString(2).trim(), resultSet.getString(3).trim())); } builder.setPartitionColumns(partitionColumns); } @@ -213,7 +231,8 @@ public class BeelineHiveClient implements IHiveClient { public static void main(String[] args) throws SQLException { - BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'"); + BeelineHiveClient loader = new BeelineHiveClient( + "-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'"); //BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " ")); HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test_kylin_fact_part"); System.out.println(hiveTableMeta); http://git-wip-us.apache.org/repos/asf/kylin/blob/55365b68/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java ---------------------------------------------------------------------- diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java new file mode 100644 index 0000000..00a25e5 --- /dev/null +++ b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.source.hive; + +import org.junit.Assert; +import org.junit.Test; + +public class BeelineHIveClientTest { + @Test + public void testBasics() { + String dataType = "varchar"; + String precision = "60"; + String scale = null; + dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale); + Assert.assertEquals("varchar(60)", dataType); + + dataType = "char"; + precision = "50"; + scale = null; + dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale); + Assert.assertEquals("char(50)", dataType); + + dataType = "decimal"; + precision = "8"; + scale = "4"; + dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale); + Assert.assertEquals("decimal(8,4)", dataType); + + dataType = "numeric"; + precision = "7"; + scale = "3"; + dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale); + Assert.assertEquals("numeric(7,3)", dataType); + } +}
