KYLIN-2515 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6cdd629 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6cdd629 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6cdd629 Branch: refs/heads/master Commit: f6cdd629817d18b60d6885753eb9681cfa4b57f2 Parents: 216ae03 Author: Yang Li <liy...@apache.org> Authored: Sun Jun 4 16:31:31 2017 +0800 Committer: nichunen <zjsy...@sjtu.org> Committed: Sun Jun 4 18:01:03 2017 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 8 +- .../apache/kylin/common/KylinConfigBase.java | 10 +- .../metadata/querymeta/ColumnMetaWithType.java | 1 + .../metadata/querymeta/SelectedColumnMeta.java | 1 + .../metadata/querymeta/TableMetaWithType.java | 1 + .../source/adhocquery/HiveAdhocConverter.java | 181 +++++++++++++++++++ .../source/adhocquery/IAdHocConverter.java | 25 +++ .../kylin/source/adhocquery/IAdHocRunner.java | 39 ++++ .../adhocquery/HiveAdhocConverterTest.java | 63 +++++++ .../kylin/storage/adhoc/AdHocRunnerBase.java | 48 ----- .../kylin/storage/adhoc/HiveAdhocConverter.java | 180 ------------------ .../kylin/storage/adhoc/IAdhocConverter.java | 25 --- .../storage/adhoc/HiveAdhocConverterTest.java | 62 ------- .../test_case_data/sandbox/kylin.properties | 8 +- .../kylin/rest/adhoc/AdHocRunnerJdbcImpl.java | 32 ++-- .../kylin/rest/adhoc/JdbcConnectionFactory.java | 5 +- .../kylin/rest/controller/QueryController.java | 5 +- .../org/apache/kylin/rest/util/AdHocUtil.java | 53 ++---- 18 files changed, 358 insertions(+), 389 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 8949a42..44a38d2 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -237,13 +237,13 @@ kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-histo #kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current ### AD-HOC QUERY ### -#kylin.query.ad-hoc.runner.class-name=org.apache.kylin.rest.adhoc.AdHocRunnerJdbcImpl +#kylin.query.ad-hoc.runner-class-name=org.apache.kylin.rest.adhoc.AdHocRunnerJdbcImpl #kylin.query.ad-hoc.jdbc.url=jdbc:hive2://sandbox:10000/default #kylin.query.ad-hoc.jdbc.driver=org.apache.hive.jdbc.HiveDriver #kylin.query.ad-hoc.jdbc.username=hive #kylin.query.ad-hoc.jdbc.password= -#kylin.query.ad-hoc.pool.max-total=8 -#kylin.query.ad-hoc.pool.max-idle=8 -#kylin.query.ad-hoc.pool.min-idle=0 +#kylin.query.ad-hoc.jdbc.pool-max-total=8 +#kylin.query.ad-hoc.jdbc.pool-max-idle=8 +#kylin.query.ad-hoc.jdbc.pool-min-idle=0 http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index f465949..c83c546 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -986,11 +986,11 @@ abstract public class KylinConfigBase implements Serializable { } public String getAdHocRunnerClassName() { - return getOptional("kylin.query.ad-hoc.runner.class-name", ""); + return getOptional("kylin.query.ad-hoc.runner-class-name", ""); } public String getAdHocConverterClassName() { - return getOptional("kylin.query.ad-hoc.converter.class-name", "org.apache.kylin.storage.adhoc.HiveAdhocConverter"); + return getOptional("kylin.query.ad-hoc.converter-class-name", "org.apache.kylin.source.adhocquery.HiveAdhocConverter"); } public String getJdbcUrl() { @@ -1010,15 +1010,15 @@ abstract public class KylinConfigBase implements Serializable { } public int getPoolMaxTotal() { - return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.pool.max-total", "8")); + return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-max-total", "8")); } public int getPoolMaxIdle() { - return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.pool.max-idle", "8")); + return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-max-idle", "8")); } public int getPoolMinIdle() { - return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.pool.min-idle", "0")); + return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-min-idle", "0")); } // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/ColumnMetaWithType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/ColumnMetaWithType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/ColumnMetaWithType.java index e3cb86b..101ebd5 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/ColumnMetaWithType.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/ColumnMetaWithType.java @@ -24,6 +24,7 @@ import java.util.HashSet; /** * Created by luwei on 17-4-26. */ +@SuppressWarnings("serial") public class ColumnMetaWithType extends ColumnMeta { public static enum columnTypeEnum implements Serializable { http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java b/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java index 9ba0da2..1cbe8ab 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java @@ -22,6 +22,7 @@ import java.io.Serializable; /** */ +@SuppressWarnings("serial") public class SelectedColumnMeta implements Serializable { public SelectedColumnMeta(boolean isAutoIncrement, boolean isCaseSensitive, boolean isSearchable, boolean isCurrency, int isNullalbe, boolean isSigned, int displaySize, String label, String name, String schemaName, String catelogName, String tableName, int precision, int scale, int columnType, String columnTypeName, boolean isReadOnly, boolean isWritable, boolean isDefinitelyWritable) { super(); http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java index 2ff21e4..e16aba6 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java @@ -24,6 +24,7 @@ import java.util.HashSet; /** * Created by luwei on 17-4-26. */ +@SuppressWarnings("serial") public class TableMetaWithType extends TableMeta { public static enum tableTypeEnum implements Serializable { http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java new file mode 100644 index 0000000..97d77bf --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.source.adhocquery; + +import java.util.HashMap; +import java.util.Map; +import java.util.Stack; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way +public class HiveAdhocConverter implements IAdHocConverter { + + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(HiveAdhocConverter.class); + + private static final Pattern EXTRACT_PATTERN = Pattern.compile("\\s+extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)", Pattern.CASE_INSENSITIVE); + private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s", Pattern.CASE_INSENSITIVE); + private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)", Pattern.CASE_INSENSITIVE); + private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE); + + public static String replaceString(String originString, String fromString, String toString) { + return originString.replace(fromString, toString); + } + + public static String extractReplace(String originString) { + Matcher extractMatcher = EXTRACT_PATTERN.matcher(originString); + String replacedString = originString; + Map<Integer, Integer> parenthesesPairs = null; + + while (extractMatcher.find()) { + if (parenthesesPairs == null) { + parenthesesPairs = findParenthesesPairs(originString); + } + + String functionStr = extractMatcher.group(2); + int startIdx = extractMatcher.end(3); + int endIdx = parenthesesPairs.get(extractMatcher.start(1)); + String extractInner = originString.substring(startIdx, endIdx); + int originStart = extractMatcher.start(0) + 1; + int originEnd = endIdx + 1; + + replacedString = replaceString(replacedString, originString.substring(originStart, originEnd), functionStr + "(" + extractInner + ")"); + } + + return replacedString; + } + + public static String castRepalce(String originString) { + Matcher castMatcher = CAST_PATTERN.matcher(originString); + String replacedString = originString; + + while (castMatcher.find()) { + String castStr = castMatcher.group(); + String type = castMatcher.group(2); + String supportedType = ""; + switch (type.toUpperCase()) { + case "INTEGER": + supportedType = "int"; + break; + case "SHORT": + supportedType = "smallint"; + break; + case "LONG": + supportedType = "bigint"; + break; + default: + supportedType = type; + } + + if (!supportedType.equals(type)) { + String replacedCastStr = castStr.replace(type, supportedType); + replacedString = replaceString(replacedString, castStr, replacedCastStr); + } + } + + return replacedString; + } + + public static String subqueryRepalce(String originString) { + Matcher subqueryMatcher = FROM_PATTERN.matcher(originString); + String replacedString = originString; + Map<Integer, Integer> parenthesesPairs = null; + + while (subqueryMatcher.find()) { + if (parenthesesPairs == null) { + parenthesesPairs = findParenthesesPairs(originString); + } + + int startIdx = subqueryMatcher.start(1); + int endIdx = parenthesesPairs.get(startIdx) + 1; + + replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx), originString.substring(startIdx, endIdx) + " as alias"); + } + + return replacedString; + } + + public static String concatReplace(String originString) { + Matcher concatMatcher = CONCAT_PATTERN.matcher(originString); + String replacedString = originString; + + while (concatMatcher.find()) { + String leftString = concatMatcher.group(1); + String rightString = concatMatcher.group(2); + replacedString = replaceString(replacedString, leftString + "||" + rightString, "concat(" + leftString + "," + rightString + ")"); + } + + return replacedString; + } + + public static String doConvert(String originStr) { + // Step1.Replace " with ` + String convertedSql = replaceString(originStr, "\"", "`"); + + // Step2.Replace extract functions + convertedSql = extractReplace(convertedSql); + + // Step3.Replace cast type string + convertedSql = castRepalce(convertedSql); + + // Step4.Replace sub query + convertedSql = subqueryRepalce(convertedSql); + + // Step5.Replace char_length with length + convertedSql = replaceString(convertedSql, "char_length", "length"); + + // Step6.Replace "||" with concat + convertedSql = concatReplace(convertedSql); + + return convertedSql; + } + + private static Map<Integer, Integer> findParenthesesPairs(String sql) { + Map<Integer, Integer> result = new HashMap<>(); + if (sql.length() > 1) { + Stack<Integer> lStack = new Stack<>(); + boolean inStrVal = false; + for (int i = 0; i < sql.length(); i++) { + switch (sql.charAt(i)) { + case '(': + if (!inStrVal) { + lStack.push(i); + } + break; + case ')': + if (!inStrVal && !lStack.empty()) { + result.put(lStack.pop(), i); + } + break; + default: + break; + } + } + } + return result; + } + + @Override + public String convert(String originSql) { + return doConvert(originSql); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java new file mode 100644 index 0000000..c4b87f8 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.source.adhocquery; + +/** + * convert the query to satisfy the parser of adhoc query engine + */ +public interface IAdHocConverter { + String convert(String originSql); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java new file mode 100644 index 0000000..369325c --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.source.adhocquery; + +import java.util.List; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; + +public interface IAdHocRunner { + + void init(KylinConfig config); + + /** + * Run an ad-hoc query in the source database in case Kylin cannot serve using cube. + * + * @param query the query statement + * @param returnRows an empty list to collect returning rows + * @param returnColumnMeta an empty list to collect metadata of returning columns + * @throws Exception if running ad-hoc query fails + */ + void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta) throws Exception; +} http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java new file mode 100644 index 0000000..85a6d61 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.source.adhocquery; + +import org.junit.Test; + +import junit.framework.TestCase; + + +public class HiveAdhocConverterTest extends TestCase { + @Test + public void testSringReplace() { + String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10"; + String replacedString = HiveAdhocConverter + .replaceString(originString, "char_length", "length"); + assertEquals(replacedString, "select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10"); + } + + @Test + public void testExtractReplace() { + String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore"; + String replacedString = HiveAdhocConverter.extractReplace(originString); + assertEquals(replacedString, "ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore"); + } + + @Test + public void testCastReplace() { + String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore"; + String replacedString = HiveAdhocConverter.castRepalce(originString); + assertEquals(replacedString, "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore"); + } + + @Test + public void testSubqueryReplace() { + String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name"; + String replacedString = HiveAdhocConverter.subqueryRepalce(originString); + assertEquals(replacedString, "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name"); + } + + @Test + public void testConcatReplace() { + String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'"; + String replacedString = HiveAdhocConverter.concatReplace(originString); + assertEquals(replacedString, "select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'"); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java b/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java deleted file mode 100644 index 7b870c6..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.adhoc; - -import java.util.List; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AdHocRunnerBase { - - private static final Logger logger = LoggerFactory.getLogger(AdHocRunnerBase.class); - - protected KylinConfig config = null; - - public AdHocRunnerBase() { - } - - public AdHocRunnerBase(KylinConfig config) { - this.config = config; - } - - public void setConfig(KylinConfig config) { - this.config = config; - } - - public abstract void init(); - - public abstract void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws Exception; -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-storage/src/main/java/org/apache/kylin/storage/adhoc/HiveAdhocConverter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/adhoc/HiveAdhocConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/adhoc/HiveAdhocConverter.java deleted file mode 100644 index 1a43557..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/adhoc/HiveAdhocConverter.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -package org.apache.kylin.storage.adhoc; - -import java.util.HashMap; -import java.util.Map; -import java.util.Stack; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way -public class HiveAdhocConverter implements IAdhocConverter { - - private static final Logger logger = LoggerFactory.getLogger(HiveAdhocConverter.class); - - private static final Pattern EXTRACT_PATTERN = Pattern.compile("\\s+extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)", Pattern.CASE_INSENSITIVE); - private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s", Pattern.CASE_INSENSITIVE); - private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)", Pattern.CASE_INSENSITIVE); - private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE); - - public static String replaceString(String originString, String fromString, String toString) { - return originString.replace(fromString, toString); - } - - public static String extractReplace(String originString) { - Matcher extractMatcher = EXTRACT_PATTERN.matcher(originString); - String replacedString = originString; - Map<Integer, Integer> parenthesesPairs = null; - - while (extractMatcher.find()) { - if (parenthesesPairs == null) { - parenthesesPairs = findParenthesesPairs(originString); - } - - String functionStr = extractMatcher.group(2); - int startIdx = extractMatcher.end(3); - int endIdx = parenthesesPairs.get(extractMatcher.start(1)); - String extractInner = originString.substring(startIdx, endIdx); - int originStart = extractMatcher.start(0) + 1; - int originEnd = endIdx + 1; - - replacedString = replaceString(replacedString, originString.substring(originStart, originEnd), functionStr + "(" + extractInner + ")"); - } - - return replacedString; - } - - public static String castRepalce(String originString) { - Matcher castMatcher = CAST_PATTERN.matcher(originString); - String replacedString = originString; - - while (castMatcher.find()) { - String castStr = castMatcher.group(); - String type = castMatcher.group(2); - String supportedType = ""; - switch (type.toUpperCase()) { - case "INTEGER": - supportedType = "int"; - break; - case "SHORT": - supportedType = "smallint"; - break; - case "LONG": - supportedType = "bigint"; - break; - default: - supportedType = type; - } - - if (!supportedType.equals(type)) { - String replacedCastStr = castStr.replace(type, supportedType); - replacedString = replaceString(replacedString, castStr, replacedCastStr); - } - } - - return replacedString; - } - - public static String subqueryRepalce(String originString) { - Matcher subqueryMatcher = FROM_PATTERN.matcher(originString); - String replacedString = originString; - Map<Integer, Integer> parenthesesPairs = null; - - while (subqueryMatcher.find()) { - if (parenthesesPairs == null) { - parenthesesPairs = findParenthesesPairs(originString); - } - - int startIdx = subqueryMatcher.start(1); - int endIdx = parenthesesPairs.get(startIdx) + 1; - - replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx), originString.substring(startIdx, endIdx) + " as alias"); - } - - return replacedString; - } - - public static String concatReplace(String originString) { - Matcher concatMatcher = CONCAT_PATTERN.matcher(originString); - String replacedString = originString; - - while (concatMatcher.find()) { - String leftString = concatMatcher.group(1); - String rightString = concatMatcher.group(2); - replacedString = replaceString(replacedString, leftString + "||" + rightString, "concat(" + leftString + "," + rightString + ")"); - } - - return replacedString; - } - - public static String doConvert(String originStr) { - // Step1.Replace " with ` - String convertedSql = replaceString(originStr, "\"", "`"); - - // Step2.Replace extract functions - convertedSql = extractReplace(convertedSql); - - // Step3.Replace cast type string - convertedSql = castRepalce(convertedSql); - - // Step4.Replace sub query - convertedSql = subqueryRepalce(convertedSql); - - // Step5.Replace char_length with length - convertedSql = replaceString(convertedSql, "char_length", "length"); - - // Step6.Replace "||" with concat - convertedSql = concatReplace(convertedSql); - - return convertedSql; - } - - private static Map<Integer, Integer> findParenthesesPairs(String sql) { - Map<Integer, Integer> result = new HashMap<>(); - if (sql.length() > 1) { - Stack<Integer> lStack = new Stack<>(); - boolean inStrVal = false; - for (int i = 0; i < sql.length(); i++) { - switch (sql.charAt(i)) { - case '(': - if (!inStrVal) { - lStack.push(i); - } - break; - case ')': - if (!inStrVal && !lStack.empty()) { - result.put(lStack.pop(), i); - } - break; - default: - break; - } - } - } - return result; - } - - @Override - public String convert(String originSql) { - return doConvert(originSql); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-storage/src/main/java/org/apache/kylin/storage/adhoc/IAdhocConverter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/adhoc/IAdhocConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/adhoc/IAdhocConverter.java deleted file mode 100644 index d5815bb..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/adhoc/IAdhocConverter.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -package org.apache.kylin.storage.adhoc; - -/** - * convert the query to satisfy the parser of adhoc query engine - */ -public interface IAdhocConverter { - String convert(String originSql); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/core-storage/src/test/java/org/apache/kylin/storage/adhoc/HiveAdhocConverterTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/adhoc/HiveAdhocConverterTest.java b/core-storage/src/test/java/org/apache/kylin/storage/adhoc/HiveAdhocConverterTest.java deleted file mode 100644 index 62f6792..0000000 --- a/core-storage/src/test/java/org/apache/kylin/storage/adhoc/HiveAdhocConverterTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.adhoc; - -import junit.framework.TestCase; -import org.junit.Test; - - -public class HiveAdhocConverterTest extends TestCase { - @Test - public void testSringReplace() { - String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10"; - String replacedString = HiveAdhocConverter - .replaceString(originString, "char_length", "length"); - assertEquals(replacedString, "select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10"); - } - - @Test - public void testExtractReplace() { - String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore"; - String replacedString = HiveAdhocConverter.extractReplace(originString); - assertEquals(replacedString, "ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore"); - } - - @Test - public void testCastReplace() { - String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore"; - String replacedString = HiveAdhocConverter.castRepalce(originString); - assertEquals(replacedString, "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore"); - } - - @Test - public void testSubqueryReplace() { - String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name"; - String replacedString = HiveAdhocConverter.subqueryRepalce(originString); - assertEquals(replacedString, "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name"); - } - - @Test - public void testConcatReplace() { - String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'"; - String replacedString = HiveAdhocConverter.concatReplace(originString); - assertEquals(replacedString, "select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'"); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 8caebc2..83a1ef3 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -187,13 +187,13 @@ kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current ### AD-HOC QUERY ### -#kylin.query.ad-hoc.runner.class-name=org.apache.kylin.rest.adhoc.AdHocRunnerJdbcImpl +#kylin.query.ad-hoc.runner-class-name=org.apache.kylin.rest.adhoc.AdHocRunnerJdbcImpl #kylin.query.ad-hoc.jdbc.url=jdbc:hive2://sandbox:10000/default #kylin.query.ad-hoc.jdbc.driver=org.apache.hive.jdbc.HiveDriver #kylin.query.ad-hoc.jdbc.username=hive #kylin.query.ad-hoc.jdbc.password= -#kylin.query.ad-hoc.pool.max-total=8 -#kylin.query.ad-hoc.pool.max-idle=8 -#kylin.query.ad-hoc.pool.min-idle=0 +#kylin.query.ad-hoc.jdbc.pool-max-total=8 +#kylin.query.ad-hoc.jdbc.pool-max-idle=8 +#kylin.query.ad-hoc.jdbc.pool-min-idle=0 http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/server-base/src/main/java/org/apache/kylin/rest/adhoc/AdHocRunnerJdbcImpl.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/adhoc/AdHocRunnerJdbcImpl.java b/server-base/src/main/java/org/apache/kylin/rest/adhoc/AdHocRunnerJdbcImpl.java index 275fce5..44d1770 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/adhoc/AdHocRunnerJdbcImpl.java +++ b/server-base/src/main/java/org/apache/kylin/rest/adhoc/AdHocRunnerJdbcImpl.java @@ -28,34 +28,26 @@ import java.util.LinkedList; import java.util.List; import org.apache.commons.pool.impl.GenericObjectPool; -import org.apache.kylin.storage.adhoc.AdHocRunnerBase; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; +import org.apache.kylin.source.adhocquery.IAdHocRunner; -public class AdHocRunnerJdbcImpl extends AdHocRunnerBase { +public class AdHocRunnerJdbcImpl implements IAdHocRunner { private static JdbcConnectionPool pool = null; - public AdHocRunnerJdbcImpl() { - super(); - } - - public AdHocRunnerJdbcImpl(KylinConfig config) { - super(config); - } - @Override - public void init() { - if (this.pool == null) { - this.pool = new JdbcConnectionPool(); - JdbcConnectionFactory factory = new JdbcConnectionFactory(this.config.getJdbcUrl(), this.config.getJdbcDriverClass(), this.config.getJdbcUsername(), this.config.getJdbcPassword()); + public void init(KylinConfig config) { + if (pool == null) { + pool = new JdbcConnectionPool(); + JdbcConnectionFactory factory = new JdbcConnectionFactory(config.getJdbcUrl(), config.getJdbcDriverClass(), config.getJdbcUsername(), config.getJdbcPassword()); GenericObjectPool.Config poolConfig = new GenericObjectPool.Config(); - poolConfig.maxActive = this.config.getPoolMaxTotal(); - poolConfig.maxIdle = this.config.getPoolMaxIdle(); - poolConfig.minIdle = this.config.getPoolMinIdle(); + poolConfig.maxActive = config.getPoolMaxTotal(); + poolConfig.maxIdle = config.getPoolMaxIdle(); + poolConfig.minIdle = config.getPoolMinIdle(); try { - this.pool.createPool(factory, poolConfig); + pool.createPool(factory, poolConfig); } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); } @@ -96,11 +88,11 @@ public class AdHocRunnerJdbcImpl extends AdHocRunnerBase { } private Connection getConnection() { - return this.pool.getConnection(); + return pool.getConnection(); } private void closeConnection(Connection connection) { - this.pool.returnConnection(connection); + pool.returnConnection(connection); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/server-base/src/main/java/org/apache/kylin/rest/adhoc/JdbcConnectionFactory.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/adhoc/JdbcConnectionFactory.java b/server-base/src/main/java/org/apache/kylin/rest/adhoc/JdbcConnectionFactory.java index 42613fe..dff98d0 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/adhoc/JdbcConnectionFactory.java +++ b/server-base/src/main/java/org/apache/kylin/rest/adhoc/JdbcConnectionFactory.java @@ -19,12 +19,13 @@ package org.apache.kylin.rest.adhoc; -import org.apache.commons.pool.PoolableObjectFactory; - import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import org.apache.commons.pool.PoolableObjectFactory; + +@SuppressWarnings("unused") class JdbcConnectionFactory implements PoolableObjectFactory { private final String jdbcUrl; http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java index f6bfe3e..0da92c7 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java @@ -26,10 +26,10 @@ import java.util.List; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; -import org.apache.kylin.rest.exception.InternalErrorException; -import org.apache.kylin.rest.model.Query; import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; import org.apache.kylin.metadata.querymeta.TableMeta; +import org.apache.kylin.rest.exception.InternalErrorException; +import org.apache.kylin.rest.model.Query; import org.apache.kylin.rest.request.MetaRequest; import org.apache.kylin.rest.request.PrepareSqlRequest; import org.apache.kylin.rest.request.SQLRequest; @@ -59,6 +59,7 @@ import org.supercsv.prefs.CsvPreference; @Controller public class QueryController extends BasicController { + @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(QueryController.class); @Autowired http://git-wip-us.apache.org/repos/asf/kylin/blob/f6cdd629/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java index 8221790..76ff237 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java +++ b/server-base/src/main/java/org/apache/kylin/rest/util/AdHocUtil.java @@ -18,8 +18,6 @@ package org.apache.kylin.rest.util; -import static org.apache.kylin.metadata.MetadataManager.CCInfo; - import java.sql.SQLException; import java.util.Collections; import java.util.List; @@ -33,15 +31,16 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.Triple; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.MetadataManager.CCInfo; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; import org.apache.kylin.query.routing.NoRealizationFoundException; -import org.apache.kylin.rest.exception.InternalErrorException; -import org.apache.kylin.storage.adhoc.AdHocRunnerBase; -import org.apache.kylin.storage.adhoc.IAdhocConverter; +import org.apache.kylin.source.adhocquery.IAdHocRunner; +import org.apache.kylin.source.adhocquery.IAdHocConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,47 +58,27 @@ public class AdHocUtil { boolean isExpectedCause = (ExceptionUtils.getRootCause(sqlException).getClass() .equals(NoRealizationFoundException.class)); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - Boolean isAdHoc = false; if (isExpectedCause && kylinConfig.isAdhocEnabled()) { - Class runnerClass = Class.forName(kylinConfig.getAdHocRunnerClassName()); - Class converterClass = Class.forName(kylinConfig.getAdHocConverterClassName()); - Object runnerObj = runnerClass.newInstance(); - Object converterObj = converterClass.newInstance(); - - if (!(runnerObj instanceof AdHocRunnerBase)) { - throw new InternalErrorException("Ad-hoc runner class should be sub-class of AdHocRunnerBase"); - } - - if (!(converterObj instanceof IAdhocConverter)) { - throw new InternalErrorException("Ad-hoc converter class should implement of IAdhocConverter"); - } + IAdHocRunner runner = (IAdHocRunner) ClassUtil.newInstance(kylinConfig.getAdHocRunnerClassName()); + IAdHocConverter converter = (IAdHocConverter) ClassUtil.newInstance(kylinConfig.getAdHocConverterClassName()); - AdHocRunnerBase runner = (AdHocRunnerBase) runnerObj; - IAdhocConverter converter = (IAdhocConverter) converterObj; - runner.setConfig(kylinConfig); + runner.init(kylinConfig); - logger.debug("Ad-hoc query enabled for Kylin"); + logger.debug("Ad-hoc query runner {}", runner); - runner.init(); - - try { - String expandCC = restoreComputedColumnToExpr(sql, project); - String adhocSql = converter.convert(expandCC); - if (!adhocSql.equals(adhocSql)) { - logger.info("before delegating to adhoc, the query is converted to {} ", adhocSql); - } - - runner.executeQuery(adhocSql, results, columnMetas); - isAdHoc = true; - } catch (Exception exception) { - throw exception; + String expandCC = restoreComputedColumnToExpr(sql, project); + String adhocSql = converter.convert(expandCC); + if (!adhocSql.equals(sql)) { + logger.info("before delegating to adhoc, the query is converted to {} ", adhocSql); } + + runner.executeQuery(adhocSql, results, columnMetas); + + return true; } else { throw sqlException; } - - return isAdHoc; } private final static Pattern identifierInSqlPattern = Pattern.compile(
