Merge commit 'a36288142f30cad931881107872adb4ba0a574a3'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e9ec116c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e9ec116c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e9ec116c Branch: refs/heads/master-hbase0.98 Commit: e9ec116c247072603a5405ddba425fd349ba670b Parents: f44f828 a362881 Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Apr 11 17:12:46 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Apr 11 17:12:46 2017 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 1 + .../kylin/common/util/HiveCmdBuilder.java | 16 +++- .../kylin/common/util/HiveCmdBuilderTest.java | 1 + .../cube/model/CubeJoinedFlatTableDesc.java | 4 +- .../org/apache/kylin/job/JoinedFlatTable.java | 34 +++---- .../kylin/job/engine/JobEngineConfig.java | 2 +- .../apache/kylin/job/JoinedFlatTableTest.java | 4 +- .../metadata/filter/CompareTupleFilter.java | 14 +++ .../filter/FilterOptimizeTransformer.java | 89 ++++++++++++++++++ .../kylin/metadata/model/FunctionDesc.java | 8 +- .../kylin/metadata/model/ParameterDesc.java | 58 ++++++------ .../apache/kylin/metadata/model/TblColRef.java | 10 ++ .../gtrecord/GTCubeStorageQueryBase.java | 2 +- .../gtrecord/StorageResponseGTScatter.java | 22 +++-- .../engine/mr/common/HadoopStatusGetter.java | 15 +-- examples/test_case_data/sandbox/hdfs-site.xml | 4 + .../test_case_data/sandbox/kylin.properties | 1 + .../src/test/resources/query/sql/query104.sql | 31 +++++++ .../resources/query/sql_casewhen/query55.sql | 22 +++++ .../query/sql_casewhen/query55.sql.disable | 22 ----- .../resources/query/sql_subquery/query15.sql | 29 ++++++ .../resources/query/sql_subquery/query16.sql | 38 ++++++++ .../kylin/query/relnode/OLAPAggregateRel.java | 13 ++- .../kylin/query/relnode/OLAPFilterRel.java | 25 ++--- .../kylin/query/relnode/OLAPProjectRel.java | 12 +-- .../kylin/query/relnode/OLAPTableScan.java | 2 +- .../org/apache/kylin/query/udf/ConcatUDF.java | 28 ++++++ .../org/apache/kylin/rest/util/QueryUtil.java | 15 --- .../apache/kylin/rest/util/QueryUtilTest.java | 6 -- .../apache/kylin/source/hive/HiveMRInput.java | 97 +++++++++----------- .../kylin/storage/hbase/HBaseConnection.java | 7 +- .../hbase/util/DeployCoprocessorCLI.java | 1 + 32 files changed, 425 insertions(+), 208 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --cc core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 7313630,5553d34..54c1e61 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@@ -19,8 -19,8 +19,9 @@@ package org.apache.kylin.job; import java.io.File; +import java.lang.reflect.Method; import java.util.HashSet; + import java.util.Map; import java.util.Set; import javax.xml.parsers.DocumentBuilder; @@@ -49,18 -46,18 +50,17 @@@ public class JoinedFlatTable return storageDfsDir + "/" + flatDesc.getTableName(); } - public static String generateHiveSetStatements(JobEngineConfig engineConfig) { - public static String generateHiveInitStatements( - String flatTableDatabase, String kylinHiveFile, Map<String, String> cubeOverrides) { ++ public static String generateHiveInitStatements(String flatTableDatabase, String kylinHiveFile, Map<String, String> cubeOverrides) { + StringBuilder buffer = new StringBuilder(); + buffer.append("USE ").append(flatTableDatabase).append(";\n"); try { - File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath()); - - if (hadoopPropertiesFile.exists()) { + File file = new File(kylinHiveFile); + if (file.exists()) { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder; - Document doc; - builder = factory.newDocumentBuilder(); - doc = builder.parse(hadoopPropertiesFile); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(file); NodeList nl = doc.getElementsByTagName("property"); for (int i = 0; i < nl.getLength(); i++) { String name = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue(); @@@ -101,24 -103,8 +106,19 @@@ return ddl.toString(); } - public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc, JobEngineConfig engineConfig) { + public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) { + final KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig(); - StringBuilder sql = new StringBuilder(); - + if (kylinConfig.isAdvancedFlatTableUsed()) { + try { + Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass()); + Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class, JobEngineConfig.class); - return (String) method.invoke(null, flatDesc, engineConfig); ++ return (String) method.invoke(null, flatDesc); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + - sql.append(generateHiveSetStatements(engineConfig)); - sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n"); - - return sql.toString(); + return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n"; } public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) { http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java ----------------------------------------------------------------------
