minor, rename adhoc to pushdown in KYLIN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7002dd86 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7002dd86 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7002dd86 Branch: refs/heads/kylin-2.1.x Commit: 7002dd864a3f43a2e97a022d1d348773609f446d Parents: d91f522 Author: Cheng Wang <cheng.w...@kyligence.io> Authored: Fri Jun 30 21:12:54 2017 +0800 Committer: liyang-gmt8 <liy...@apache.org> Committed: Fri Jun 30 21:18:10 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 28 +- .../main/resources/kylin-defaults.properties | 18 +- .../kylin/metadata/project/ProjectManager.java | 8 +- .../source/adhocquery/HiveAdhocConverter.java | 297 ------------------- .../adhocquery/HivePushDownConverter.java | 297 +++++++++++++++++++ .../source/adhocquery/IAdHocConverter.java | 25 -- .../kylin/source/adhocquery/IAdHocRunner.java | 39 --- .../source/adhocquery/IPushDownConverter.java | 25 ++ .../source/adhocquery/IPushDownRunner.java | 39 +++ .../adhocquery/HiveAdhocConverterTest.java | 78 ----- .../adhocquery/HivePushDownConverterTest.java | 78 +++++ .../test_case_data/sandbox/kylin.properties | 18 +- .../org/apache/kylin/query/KylinTestBase.java | 4 +- .../kylin/query/adhoc/AdHocRunnerJdbcImpl.java | 122 -------- .../query/adhoc/PushDownRunnerJdbcImpl.java | 122 ++++++++ .../apache/kylin/rest/response/SQLResponse.java | 17 +- .../apache/kylin/rest/service/QueryService.java | 150 +++++++--- .../org/apache/kylin/rest/util/AdHocUtil.java | 178 ----------- .../apache/kylin/rest/util/PushDownUtil.java | 177 +++++++++++ .../apache/kylin/rest/util/AdHocUtilTest.java | 94 ------ .../kylin/rest/util/PushDownUtilTest.java | 94 ++++++ 21 files changed, 986 insertions(+), 922 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 4a21ded..179d61f 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1022,45 +1022,45 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0")); } - public boolean isAdhocEnabled() { - return StringUtils.isNotEmpty(getAdHocRunnerClassName()); + public boolean isPushDownEnabled() { + return StringUtils.isNotEmpty(getPushDownRunnerClassName()); } - public String getAdHocRunnerClassName() { - return getOptional("kylin.query.ad-hoc.runner-class-name", ""); + public String getPushDownRunnerClassName() { + return getOptional("kylin.query.pushdown.runner-class-name", ""); } - public String getAdHocConverterClassName() { - return getOptional("kylin.query.ad-hoc.converter-class-name", - "org.apache.kylin.source.adhocquery.HiveAdhocConverter"); + public String getPushDownConverterClassName() { + return getOptional("kylin.query.pushdown.converter-class-name", + "org.apache.kylin.source.adhocquery.HivePushDownConverter"); } public String getJdbcUrl() { - return getOptional("kylin.query.ad-hoc.jdbc.url", ""); + return getOptional("kylin.query.pushdown.jdbc.url", ""); } public String getJdbcDriverClass() { - return getOptional("kylin.query.ad-hoc.jdbc.driver", ""); + return getOptional("kylin.query.pushdown.jdbc.driver", ""); } public String getJdbcUsername() { - return getOptional("kylin.query.ad-hoc.jdbc.username", ""); + return getOptional("kylin.query.pushdown.jdbc.username", ""); } public String getJdbcPassword() { - return getOptional("kylin.query.ad-hoc.jdbc.password", ""); + return getOptional("kylin.query.pushdown.jdbc.password", ""); } public int getPoolMaxTotal() { - return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-max-total", "8")); + return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-max-total", "8")); } public int getPoolMaxIdle() { - return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-max-idle", "8")); + return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-max-idle", "8")); } public int getPoolMinIdle() { - return Integer.parseInt(this.getOptional("kylin.query.ad-hoc.jdbc.pool-min-idle", "0")); + return Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-min-idle", "0")); } // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-common/src/main/resources/kylin-defaults.properties ---------------------------------------------------------------------- diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties index beac63c..cb511e7 100644 --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@ -237,15 +237,15 @@ kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-histo #kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -### AD-HOC QUERY ### +### QUERY PUSH DOWN ### -#kylin.query.ad-hoc.runner-class-name=org.apache.kylin.query.adhoc.AdHocRunnerJdbcImpl +#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl -#kylin.query.ad-hoc.jdbc.url=jdbc:hive2://sandbox:10000/default -#kylin.query.ad-hoc.jdbc.driver=org.apache.hive.jdbc.HiveDriver -#kylin.query.ad-hoc.jdbc.username=hive -#kylin.query.ad-hoc.jdbc.password= +#kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:10000/default +#kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver +#kylin.query.pushdown.jdbc.username=hive +#kylin.query.pushdown.jdbc.password= -#kylin.query.ad-hoc.jdbc.pool-max-total=8 -#kylin.query.ad-hoc.jdbc.pool-max-idle=8 -#kylin.query.ad-hoc.jdbc.pool-min-idle=0 +#kylin.query.pushdown.jdbc.pool-max-total=8 +#kylin.query.pushdown.jdbc.pool-max-idle=8 +#kylin.query.pushdown.jdbc.pool-min-idle=0 http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index ea03c3c..b801d8c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -437,7 +437,7 @@ public class ProjectManager { } public Collection<TableDesc> listExposedTables(String project) { - return config.isAdhocEnabled() ? // + return config.isPushDownEnabled() ? // this.listDefinedTables(project) : // l2Cache.listExposedTables(norm(project)); } @@ -445,7 +445,7 @@ public class ProjectManager { public List<ColumnDesc> listExposedColumns(String project, TableDesc tableDesc) { Set<ColumnDesc> exposedColumns = l2Cache.listExposedColumns(norm(project), tableDesc.getIdentity()); - if (config.isAdhocEnabled()) { + if (config.isPushDownEnabled()) { // take care of computed columns Set<ColumnDesc> dedup = Sets.newHashSet(tableDesc.getColumns()); dedup.addAll(exposedColumns); @@ -456,13 +456,13 @@ public class ProjectManager { } public boolean isExposedTable(String project, String table) { - return config.isAdhocEnabled() ? // + return config.isPushDownEnabled() ? // l2Cache.isDefinedTable(norm(project), table) : // l2Cache.isExposedTable(norm(project), table); } public boolean isExposedColumn(String project, String table, String col) { - return config.isAdhocEnabled() ? // + return config.isPushDownEnabled() ? // l2Cache.isDefinedColumn(norm(project), table, col) || l2Cache.isExposedColumn(norm(project), table, col) : // l2Cache.isExposedColumn(norm(project), table, col); http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java deleted file mode 100644 index e8c06ff..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -package org.apache.kylin.source.adhocquery; - -import java.util.HashMap; -import java.util.Map; -import java.util.Stack; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.ImmutableSet; - -//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way -public class HiveAdhocConverter implements IAdHocConverter { - - @SuppressWarnings("unused") - private static final Logger logger = LoggerFactory.getLogger(HiveAdhocConverter.class); - - private static final Pattern EXTRACT_PATTERN = Pattern.compile("extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)", - Pattern.CASE_INSENSITIVE); - private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s", - Pattern.CASE_INSENSITIVE); - private static final Pattern ALIAS_PATTERN = Pattern.compile("\\s+([`'_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE); - private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)", - Pattern.CASE_INSENSITIVE); - private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)", - Pattern.CASE_INSENSITIVE); - private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,", - Pattern.CASE_INSENSITIVE); - private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("A", "ABS", "ABSOLUTE", "ACTION", - "ADA", "ADD", "ADMIN", "AFTER", "ALL", "ALLOCATE", "ALLOW", "ALTER", "ALWAYS", "AND", "ANY", "APPLY", "ARE", - "ARRAY", "ARRAY_MAX_CARDINALITY", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT", - "ATOMIC", "ATTRIBUTE", "ATTRIBUTES", "AUTHORIZATION", "AVG", "BEFORE", "BEGIN", "BEGIN_FRAME", - "BEGIN_PARTITION", "BERNOULLI", "BETWEEN", "BIGINT", "BINARY", "BIT", "BLOB", "BOOLEAN", "BOTH", "BREADTH", - "BY", "C", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG", - "CATALOG_NAME", "CEIL", "CEILING", "CENTURY", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTERS", - "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH", - "CHECK", "CLASSIFIER", "CLASS_ORIGIN", "CLOB", "CLOSE", "COALESCE", "COBOL", "COLLATE", "COLLATION", - "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLLECT", "COLUMN", "COLUMN_NAME", - "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMIT", "COMMITTED", "CONDITION", "CONDITION_NUMBER", - "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG", - "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERT", "CORR", - "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "CROSS", "CUBE", "CUME_DIST", "CURRENT", - "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE", - "CURRENT_ROW", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE", - "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE", - "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECADE", "DECIMAL", "DECLARE", "DEFAULT", - "DEFAULTS", "DEFERRABLE", "DEFERRED", "DEFINE", "DEFINED", "DEFINER", "DEGREE", "DELETE", "DENSE_RANK", - "DEPTH", "DEREF", "DERIVED", "DESC", "DESCRIBE", "DESCRIPTION", "DESCRIPTOR", "DETERMINISTIC", - "DIAGNOSTICS", "DISALLOW", "DISCONNECT", "DISPATCH", "DISTINCT", "DOMAIN", "DOUBLE", "DOW", "DOY", "DROP", - "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELEMENT", "ELSE", "EMPTY", "END", - "END-EXEC", "END_FRAME", "END_PARTITION", "EPOCH", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION", - "EXCLUDE", "EXCLUDING", "EXEC", "EXECUTE", "EXISTS", "EXP", "EXPLAIN", "EXTEND", "EXTERNAL", "EXTRACT", - "FALSE", "FETCH", "FILTER", "FINAL", "FIRST", "FIRST_VALUE", "FLOAT", "FLOOR", "FOLLOWING", "FOR", - "FOREIGN", "FORTRAN", "FOUND", "FRAC_SECOND", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION", - "G", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING", - "GROUPS", "HAVING", "HIERARCHY", "HOLD", "HOUR", "IDENTITY", "IMMEDIATE", "IMMEDIATELY", "IMPLEMENTATION", - "IMPORT", "IN", "INCLUDING", "INCREMENT", "INDICATOR", "INITIAL", "INITIALLY", "INNER", "INOUT", "INPUT", - "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INT", "INTEGER", "INTERSECT", "INTERSECTION", - "INTERVAL", "INTO", "INVOKER", "IS", "ISOLATION", "JAVA", "JOIN", "JSON", "K", "KEY", "KEY_MEMBER", - "KEY_TYPE", "LABEL", "LAG", "LANGUAGE", "LARGE", "LAST", "LAST_VALUE", "LATERAL", "LEAD", "LEADING", "LEFT", - "LENGTH", "LEVEL", "LIBRARY", "LIKE", "LIKE_REGEX", "LIMIT", "LN", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP", - "LOCATOR", "LOWER", "M", "MAP", "MATCH", "MATCHED", "MATCHES", "MATCH_NUMBER", "MATCH_RECOGNIZE", "MAX", - "MAXVALUE", "MEASURES", "MEMBER", "MERGE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT", - "METHOD", "MICROSECOND", "MILLENNIUM", "MIN", "MINUS", "MINUTE", "MINVALUE", "MOD", "MODIFIES", "MODULE", - "MONTH", "MORE", "MULTISET", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NESTING", - "NEW", "NEXT", "NO", "NONE", "NORMALIZE", "NORMALIZED", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLABLE", - "NULLIF", "NULLS", "NUMBER", "NUMERIC", "OBJECT", "OCCURRENCES_REGEX", "OCTETS", "OCTET_LENGTH", "OF", - "OFFSET", "OLD", "OMIT", "ON", "ONE", "ONLY", "OPEN", "OPTION", "OPTIONS", "OR", "ORDER", "ORDERING", - "ORDINALITY", "OTHERS", "OUT", "OUTER", "OUTPUT", "OVER", "OVERLAPS", "OVERLAY", "OVERRIDING", "PAD", - "PARAMETER", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG", - "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PARTITION", "PASCAL", "PASSTHROUGH", - "PAST", "PATH", "PATTERN", "PER", "PERCENT", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERCENT_RANK", "PERIOD", - "PERMUTE", "PLACING", "PLAN", "PLI", "PORTION", "POSITION", "POSITION_REGEX", "POWER", "PRECEDES", - "PRECEDING", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURE", - "PUBLIC", "QUARTER", "RANGE", "RANK", "READ", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES", - "REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "REGR_SLOPE", - "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELATIVE", "RELEASE", "REPEATABLE", "REPLACE", "RESET", "RESTART", - "RESTRICT", "RESULT", "RETURN", "RETURNED_CARDINALITY", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH", - "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE", - "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "ROW_NUMBER", "RUNNING", - "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCOPE_CATALOGS", "SCOPE_NAME", "SCOPE_SCHEMA", - "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SEEK", "SELECT", "SELF", "SENSITIVE", "SEQUENCE", - "SERIALIZABLE", "SERVER", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETS", "SHOW", "SIMILAR", - "SIMPLE", "SIZE", "SKIP", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE", - "SPECIFIC_NAME", "SQL", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "SQL_BIGINT", "SQL_BINARY", "SQL_BIT", - "SQL_BLOB", "SQL_BOOLEAN", "SQL_CHAR", "SQL_CLOB", "SQL_DATE", "SQL_DECIMAL", "SQL_DOUBLE", "SQL_FLOAT", - "SQL_INTEGER", "SQL_INTERVAL_DAY", "SQL_INTERVAL_DAY_TO_HOUR", "SQL_INTERVAL_DAY_TO_MINUTE", - "SQL_INTERVAL_DAY_TO_SECOND", "SQL_INTERVAL_HOUR", "SQL_INTERVAL_HOUR_TO_MINUTE", - "SQL_INTERVAL_HOUR_TO_SECOND", "SQL_INTERVAL_MINUTE", "SQL_INTERVAL_MINUTE_TO_SECOND", "SQL_INTERVAL_MONTH", - "SQL_INTERVAL_SECOND", "SQL_INTERVAL_YEAR", "SQL_INTERVAL_YEAR_TO_MONTH", "SQL_LONGVARBINARY", - "SQL_LONGVARCHAR", "SQL_LONGVARNCHAR", "SQL_NCHAR", "SQL_NCLOB", "SQL_NUMERIC", "SQL_NVARCHAR", "SQL_REAL", - "SQL_SMALLINT", "SQL_TIME", "SQL_TIMESTAMP", "SQL_TINYINT", "SQL_TSI_DAY", "SQL_TSI_FRAC_SECOND", - "SQL_TSI_HOUR", "SQL_TSI_MICROSECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER", - "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SQL_VARBINARY", "SQL_VARCHAR", "SQRT", "START", "STATE", - "STATEMENT", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "STREAM", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN", - "SUBMULTISET", "SUBSET", "SUBSTITUTE", "SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC", - "SYSTEM", "SYSTEM_TIME", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TABLE_NAME", "TEMPORARY", "THEN", "TIES", - "TIME", "TIMESTAMP", "TIMESTAMPADD", "TIMESTAMPDIFF", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TINYINT", "TO", - "TOP_LEVEL_COUNT", "TRAILING", "TRANSACTION", "TRANSACTIONS_ACTIVE", "TRANSACTIONS_COMMITTED", - "TRANSACTIONS_ROLLED_BACK", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION", - "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRIM_ARRAY", "TRUE", - "TRUNCATE", "TYPE", "UESCAPE", "UNBOUNDED", "UNCOMMITTED", "UNDER", "UNION", "UNIQUE", "UNKNOWN", "UNNAMED", - "UNNEST", "UPDATE", "UPPER", "UPSERT", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG", - "USER_DEFINED_TYPE_CODE", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VALUE", "VALUES", - "VALUE_OF", "VARBINARY", "VARCHAR", "VARYING", "VAR_POP", "VAR_SAMP", "VERSION", "VERSIONING", "VIEW", - "WEEK", "WHEN", "WHENEVER", "WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "WORK", - "WRAPPER", "WRITE", "XML", "YEAR", "ZONE"); - - public static String replaceString(String originString, String fromString, String toString) { - return originString.replace(fromString, toString); - } - - public static String extractReplace(String originString) { - Matcher extractMatcher = EXTRACT_PATTERN.matcher(originString); - String replacedString = originString; - Map<Integer, Integer> parenthesesPairs = null; - - while (extractMatcher.find()) { - if (parenthesesPairs == null) { - parenthesesPairs = findParenthesesPairs(originString); - } - - String functionStr = extractMatcher.group(2); - int startIdx = extractMatcher.end(3); - int endIdx = parenthesesPairs.get(extractMatcher.start(1)); - String extractInner = originString.substring(startIdx, endIdx); - int originStart = extractMatcher.start(0); - int originEnd = endIdx + 1; - - replacedString = replaceString(replacedString, originString.substring(originStart, originEnd), - functionStr + "(" + extractInner + ")"); - } - - return replacedString; - } - - public static String castReplace(String originString) { - Matcher castMatcher = CAST_PATTERN.matcher(originString); - String replacedString = originString; - - while (castMatcher.find()) { - String castStr = castMatcher.group(); - String type = castMatcher.group(2); - String supportedType = ""; - switch (type.toUpperCase()) { - case "INTEGER": - supportedType = "int"; - break; - case "SHORT": - supportedType = "smallint"; - break; - case "LONG": - supportedType = "bigint"; - break; - default: - supportedType = type; - } - - if (!supportedType.equals(type)) { - String replacedCastStr = castStr.replace(type, supportedType); - replacedString = replaceString(replacedString, castStr, replacedCastStr); - } - } - - return replacedString; - } - - public static String subqueryReplace(String originString) { - Matcher subqueryMatcher = FROM_PATTERN.matcher(originString); - String replacedString = originString; - Map<Integer, Integer> parenthesesPairs = null; - - while (subqueryMatcher.find()) { - if (parenthesesPairs == null) { - parenthesesPairs = findParenthesesPairs(originString); - } - - int startIdx = subqueryMatcher.start(1); - int endIdx = parenthesesPairs.get(startIdx) + 1; - - Matcher aliasMatcher = ALIAS_PATTERN.matcher(originString.substring(endIdx)); - if (aliasMatcher.find()) { - String aliasCandidate = aliasMatcher.group(1); - - if (aliasCandidate != null && !sqlKeyWordsExceptAS.contains(aliasCandidate.toUpperCase())) { - continue; - } - - replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx), - originString.substring(startIdx, endIdx) + " as alias"); - } - } - - return replacedString; - } - - public static String timestampaddReplace(String originString) { - Matcher timestampaddMatcher = TIMESTAMPADD_PATTERN.matcher(originString); - String replacedString = originString; - - while (timestampaddMatcher.find()) { - String interval = timestampaddMatcher.group(1); - String timestampaddStr = replaceString(timestampaddMatcher.group(), interval, "'" + interval + "'"); - replacedString = replaceString(replacedString, timestampaddMatcher.group(), timestampaddStr); - } - - return replacedString; - } - - public static String concatReplace(String originString) { - Matcher concatMatcher = CONCAT_PATTERN.matcher(originString); - String replacedString = originString; - - while (concatMatcher.find()) { - String leftString = concatMatcher.group(1); - String rightString = concatMatcher.group(2); - replacedString = replaceString(replacedString, leftString + "||" + rightString, - "concat(" + leftString + "," + rightString + ")"); - } - - return replacedString; - } - - public static String doConvert(String originStr) { - // Step1.Replace " with ` - String convertedSql = replaceString(originStr, "\"", "`"); - - // Step2.Replace extract functions - convertedSql = extractReplace(convertedSql); - - // Step3.Replace cast type string - convertedSql = castReplace(convertedSql); - - // Step4.Replace sub query - convertedSql = subqueryReplace(convertedSql); - - // Step5.Replace char_length with length - convertedSql = replaceString(convertedSql, "char_length", "length"); - - // Step6.Replace "||" with concat - convertedSql = concatReplace(convertedSql); - - // Step7.Add quote for interval in timestampadd - convertedSql = timestampaddReplace(convertedSql); - - return convertedSql; - } - - private static Map<Integer, Integer> findParenthesesPairs(String sql) { - Map<Integer, Integer> result = new HashMap<>(); - if (sql.length() > 1) { - Stack<Integer> lStack = new Stack<>(); - boolean inStrVal = false; - for (int i = 0; i < sql.length(); i++) { - switch (sql.charAt(i)) { - case '(': - if (!inStrVal) { - lStack.push(i); - } - break; - case ')': - if (!inStrVal && !lStack.empty()) { - result.put(lStack.pop(), i); - } - break; - default: - break; - } - } - } - return result; - } - - @Override - public String convert(String originSql) { - return doConvert(originSql); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java new file mode 100644 index 0000000..8d89294 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.source.adhocquery; + +import java.util.HashMap; +import java.util.Map; +import java.util.Stack; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableSet; + +//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way +public class HivePushDownConverter implements IPushDownConverter { + + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(HivePushDownConverter.class); + + private static final Pattern EXTRACT_PATTERN = Pattern.compile("extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)", + Pattern.CASE_INSENSITIVE); + private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s", + Pattern.CASE_INSENSITIVE); + private static final Pattern ALIAS_PATTERN = Pattern.compile("\\s+([`'_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE); + private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)", + Pattern.CASE_INSENSITIVE); + private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)", + Pattern.CASE_INSENSITIVE); + private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,", + Pattern.CASE_INSENSITIVE); + private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("A", "ABS", "ABSOLUTE", "ACTION", + "ADA", "ADD", "ADMIN", "AFTER", "ALL", "ALLOCATE", "ALLOW", "ALTER", "ALWAYS", "AND", "ANY", "APPLY", "ARE", + "ARRAY", "ARRAY_MAX_CARDINALITY", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT", + "ATOMIC", "ATTRIBUTE", "ATTRIBUTES", "AUTHORIZATION", "AVG", "BEFORE", "BEGIN", "BEGIN_FRAME", + "BEGIN_PARTITION", "BERNOULLI", "BETWEEN", "BIGINT", "BINARY", "BIT", "BLOB", "BOOLEAN", "BOTH", "BREADTH", + "BY", "C", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG", + "CATALOG_NAME", "CEIL", "CEILING", "CENTURY", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTERS", + "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH", + "CHECK", "CLASSIFIER", "CLASS_ORIGIN", "CLOB", "CLOSE", "COALESCE", "COBOL", "COLLATE", "COLLATION", + "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLLECT", "COLUMN", "COLUMN_NAME", + "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMIT", "COMMITTED", "CONDITION", "CONDITION_NUMBER", + "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG", + "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERT", "CORR", + "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "CROSS", "CUBE", "CUME_DIST", "CURRENT", + "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE", + "CURRENT_ROW", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE", + "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE", + "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECADE", "DECIMAL", "DECLARE", "DEFAULT", + "DEFAULTS", "DEFERRABLE", "DEFERRED", "DEFINE", "DEFINED", "DEFINER", "DEGREE", "DELETE", "DENSE_RANK", + "DEPTH", "DEREF", "DERIVED", "DESC", "DESCRIBE", "DESCRIPTION", "DESCRIPTOR", "DETERMINISTIC", + "DIAGNOSTICS", "DISALLOW", "DISCONNECT", "DISPATCH", "DISTINCT", "DOMAIN", "DOUBLE", "DOW", "DOY", "DROP", + "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELEMENT", "ELSE", "EMPTY", "END", + "END-EXEC", "END_FRAME", "END_PARTITION", "EPOCH", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION", + "EXCLUDE", "EXCLUDING", "EXEC", "EXECUTE", "EXISTS", "EXP", "EXPLAIN", "EXTEND", "EXTERNAL", "EXTRACT", + "FALSE", "FETCH", "FILTER", "FINAL", "FIRST", "FIRST_VALUE", "FLOAT", "FLOOR", "FOLLOWING", "FOR", + "FOREIGN", "FORTRAN", "FOUND", "FRAC_SECOND", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION", + "G", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING", + "GROUPS", "HAVING", "HIERARCHY", "HOLD", "HOUR", "IDENTITY", "IMMEDIATE", "IMMEDIATELY", "IMPLEMENTATION", + "IMPORT", "IN", "INCLUDING", "INCREMENT", "INDICATOR", "INITIAL", "INITIALLY", "INNER", "INOUT", "INPUT", + "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INT", "INTEGER", "INTERSECT", "INTERSECTION", + "INTERVAL", "INTO", "INVOKER", "IS", "ISOLATION", "JAVA", "JOIN", "JSON", "K", "KEY", "KEY_MEMBER", + "KEY_TYPE", "LABEL", "LAG", "LANGUAGE", "LARGE", "LAST", "LAST_VALUE", "LATERAL", "LEAD", "LEADING", "LEFT", + "LENGTH", "LEVEL", "LIBRARY", "LIKE", "LIKE_REGEX", "LIMIT", "LN", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP", + "LOCATOR", "LOWER", "M", "MAP", "MATCH", "MATCHED", "MATCHES", "MATCH_NUMBER", "MATCH_RECOGNIZE", "MAX", + "MAXVALUE", "MEASURES", "MEMBER", "MERGE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT", + "METHOD", "MICROSECOND", "MILLENNIUM", "MIN", "MINUS", "MINUTE", "MINVALUE", "MOD", "MODIFIES", "MODULE", + "MONTH", "MORE", "MULTISET", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NESTING", + "NEW", "NEXT", "NO", "NONE", "NORMALIZE", "NORMALIZED", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLABLE", + "NULLIF", "NULLS", "NUMBER", "NUMERIC", "OBJECT", "OCCURRENCES_REGEX", "OCTETS", "OCTET_LENGTH", "OF", + "OFFSET", "OLD", "OMIT", "ON", "ONE", "ONLY", "OPEN", "OPTION", "OPTIONS", "OR", "ORDER", "ORDERING", + "ORDINALITY", "OTHERS", "OUT", "OUTER", "OUTPUT", "OVER", "OVERLAPS", "OVERLAY", "OVERRIDING", "PAD", + "PARAMETER", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG", + "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PARTITION", "PASCAL", "PASSTHROUGH", + "PAST", "PATH", "PATTERN", "PER", "PERCENT", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERCENT_RANK", "PERIOD", + "PERMUTE", "PLACING", "PLAN", "PLI", "PORTION", "POSITION", "POSITION_REGEX", "POWER", "PRECEDES", + "PRECEDING", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURE", + "PUBLIC", "QUARTER", "RANGE", "RANK", "READ", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES", + "REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "REGR_SLOPE", + "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELATIVE", "RELEASE", "REPEATABLE", "REPLACE", "RESET", "RESTART", + "RESTRICT", "RESULT", "RETURN", "RETURNED_CARDINALITY", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH", + "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE", + "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "ROW_NUMBER", "RUNNING", + "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCOPE_CATALOGS", "SCOPE_NAME", "SCOPE_SCHEMA", + "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SEEK", "SELECT", "SELF", "SENSITIVE", "SEQUENCE", + "SERIALIZABLE", "SERVER", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETS", "SHOW", "SIMILAR", + "SIMPLE", "SIZE", "SKIP", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE", + "SPECIFIC_NAME", "SQL", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "SQL_BIGINT", "SQL_BINARY", "SQL_BIT", + "SQL_BLOB", "SQL_BOOLEAN", "SQL_CHAR", "SQL_CLOB", "SQL_DATE", "SQL_DECIMAL", "SQL_DOUBLE", "SQL_FLOAT", + "SQL_INTEGER", "SQL_INTERVAL_DAY", "SQL_INTERVAL_DAY_TO_HOUR", "SQL_INTERVAL_DAY_TO_MINUTE", + "SQL_INTERVAL_DAY_TO_SECOND", "SQL_INTERVAL_HOUR", "SQL_INTERVAL_HOUR_TO_MINUTE", + "SQL_INTERVAL_HOUR_TO_SECOND", "SQL_INTERVAL_MINUTE", "SQL_INTERVAL_MINUTE_TO_SECOND", "SQL_INTERVAL_MONTH", + "SQL_INTERVAL_SECOND", "SQL_INTERVAL_YEAR", "SQL_INTERVAL_YEAR_TO_MONTH", "SQL_LONGVARBINARY", + "SQL_LONGVARCHAR", "SQL_LONGVARNCHAR", "SQL_NCHAR", "SQL_NCLOB", "SQL_NUMERIC", "SQL_NVARCHAR", "SQL_REAL", + "SQL_SMALLINT", "SQL_TIME", "SQL_TIMESTAMP", "SQL_TINYINT", "SQL_TSI_DAY", "SQL_TSI_FRAC_SECOND", + "SQL_TSI_HOUR", "SQL_TSI_MICROSECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER", + "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SQL_VARBINARY", "SQL_VARCHAR", "SQRT", "START", "STATE", + "STATEMENT", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "STREAM", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN", + "SUBMULTISET", "SUBSET", "SUBSTITUTE", "SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC", + "SYSTEM", "SYSTEM_TIME", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TABLE_NAME", "TEMPORARY", "THEN", "TIES", + "TIME", "TIMESTAMP", "TIMESTAMPADD", "TIMESTAMPDIFF", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TINYINT", "TO", + "TOP_LEVEL_COUNT", "TRAILING", "TRANSACTION", "TRANSACTIONS_ACTIVE", "TRANSACTIONS_COMMITTED", + "TRANSACTIONS_ROLLED_BACK", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION", + "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRIM_ARRAY", "TRUE", + "TRUNCATE", "TYPE", "UESCAPE", "UNBOUNDED", "UNCOMMITTED", "UNDER", "UNION", "UNIQUE", "UNKNOWN", "UNNAMED", + "UNNEST", "UPDATE", "UPPER", "UPSERT", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG", + "USER_DEFINED_TYPE_CODE", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VALUE", "VALUES", + "VALUE_OF", "VARBINARY", "VARCHAR", "VARYING", "VAR_POP", "VAR_SAMP", "VERSION", "VERSIONING", "VIEW", + "WEEK", "WHEN", "WHENEVER", "WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "WORK", + "WRAPPER", "WRITE", "XML", "YEAR", "ZONE"); + + public static String replaceString(String originString, String fromString, String toString) { + return originString.replace(fromString, toString); + } + + public static String extractReplace(String originString) { + Matcher extractMatcher = EXTRACT_PATTERN.matcher(originString); + String replacedString = originString; + Map<Integer, Integer> parenthesesPairs = null; + + while (extractMatcher.find()) { + if (parenthesesPairs == null) { + parenthesesPairs = findParenthesesPairs(originString); + } + + String functionStr = extractMatcher.group(2); + int startIdx = extractMatcher.end(3); + int endIdx = parenthesesPairs.get(extractMatcher.start(1)); + String extractInner = originString.substring(startIdx, endIdx); + int originStart = extractMatcher.start(0); + int originEnd = endIdx + 1; + + replacedString = replaceString(replacedString, originString.substring(originStart, originEnd), + functionStr + "(" + extractInner + ")"); + } + + return replacedString; + } + + public static String castReplace(String originString) { + Matcher castMatcher = CAST_PATTERN.matcher(originString); + String replacedString = originString; + + while (castMatcher.find()) { + String castStr = castMatcher.group(); + String type = castMatcher.group(2); + String supportedType = ""; + switch (type.toUpperCase()) { + case "INTEGER": + supportedType = "int"; + break; + case "SHORT": + supportedType = "smallint"; + break; + case "LONG": + supportedType = "bigint"; + break; + default: + supportedType = type; + } + + if (!supportedType.equals(type)) { + String replacedCastStr = castStr.replace(type, supportedType); + replacedString = replaceString(replacedString, castStr, replacedCastStr); + } + } + + return replacedString; + } + + public static String subqueryReplace(String originString) { + Matcher subqueryMatcher = FROM_PATTERN.matcher(originString); + String replacedString = originString; + Map<Integer, Integer> parenthesesPairs = null; + + while (subqueryMatcher.find()) { + if (parenthesesPairs == null) { + parenthesesPairs = findParenthesesPairs(originString); + } + + int startIdx = subqueryMatcher.start(1); + int endIdx = parenthesesPairs.get(startIdx) + 1; + + Matcher aliasMatcher = ALIAS_PATTERN.matcher(originString.substring(endIdx)); + if (aliasMatcher.find()) { + String aliasCandidate = aliasMatcher.group(1); + + if (aliasCandidate != null && !sqlKeyWordsExceptAS.contains(aliasCandidate.toUpperCase())) { + continue; + } + + replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx), + originString.substring(startIdx, endIdx) + " as alias"); + } + } + + return replacedString; + } + + public static String timestampaddReplace(String originString) { + Matcher timestampaddMatcher = TIMESTAMPADD_PATTERN.matcher(originString); + String replacedString = originString; + + while (timestampaddMatcher.find()) { + String interval = timestampaddMatcher.group(1); + String timestampaddStr = replaceString(timestampaddMatcher.group(), interval, "'" + interval + "'"); + replacedString = replaceString(replacedString, timestampaddMatcher.group(), timestampaddStr); + } + + return replacedString; + } + + public static String concatReplace(String originString) { + Matcher concatMatcher = CONCAT_PATTERN.matcher(originString); + String replacedString = originString; + + while (concatMatcher.find()) { + String leftString = concatMatcher.group(1); + String rightString = concatMatcher.group(2); + replacedString = replaceString(replacedString, leftString + "||" + rightString, + "concat(" + leftString + "," + rightString + ")"); + } + + return replacedString; + } + + public static String doConvert(String originStr) { + // Step1.Replace " with ` + String convertedSql = replaceString(originStr, "\"", "`"); + + // Step2.Replace extract functions + convertedSql = extractReplace(convertedSql); + + // Step3.Replace cast type string + convertedSql = castReplace(convertedSql); + + // Step4.Replace sub query + convertedSql = subqueryReplace(convertedSql); + + // Step5.Replace char_length with length + convertedSql = replaceString(convertedSql, "char_length", "length"); + + // Step6.Replace "||" with concat + convertedSql = concatReplace(convertedSql); + + // Step7.Add quote for interval in timestampadd + convertedSql = timestampaddReplace(convertedSql); + + return convertedSql; + } + + private static Map<Integer, Integer> findParenthesesPairs(String sql) { + Map<Integer, Integer> result = new HashMap<>(); + if (sql.length() > 1) { + Stack<Integer> lStack = new Stack<>(); + boolean inStrVal = false; + for (int i = 0; i < sql.length(); i++) { + switch (sql.charAt(i)) { + case '(': + if (!inStrVal) { + lStack.push(i); + } + break; + case ')': + if (!inStrVal && !lStack.empty()) { + result.put(lStack.pop(), i); + } + break; + default: + break; + } + } + } + return result; + } + + @Override + public String convert(String originSql) { + return doConvert(originSql); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java deleted file mode 100644 index c4b87f8..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocConverter.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -package org.apache.kylin.source.adhocquery; - -/** - * convert the query to satisfy the parser of adhoc query engine - */ -public interface IAdHocConverter { - String convert(String originSql); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java deleted file mode 100644 index 369325c..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IAdHocRunner.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.source.adhocquery; - -import java.util.List; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; - -public interface IAdHocRunner { - - void init(KylinConfig config); - - /** - * Run an ad-hoc query in the source database in case Kylin cannot serve using cube. - * - * @param query the query statement - * @param returnRows an empty list to collect returning rows - * @param returnColumnMeta an empty list to collect metadata of returning columns - * @throws Exception if running ad-hoc query fails - */ - void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta) throws Exception; -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java new file mode 100644 index 0000000..c4e7515 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownConverter.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.source.adhocquery; + +/** + * convert the query to satisfy the parser of push down query engine + */ +public interface IPushDownConverter { + String convert(String originSql); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java new file mode 100644 index 0000000..c8d18aa --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.source.adhocquery; + +import java.util.List; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; + +public interface IPushDownRunner { + + void init(KylinConfig config); + + /** + * Run an pushdown query in the source database in case Kylin cannot serve using cube. + * + * @param query the query statement + * @param returnRows an empty list to collect returning rows + * @param returnColumnMeta an empty list to collect metadata of returning columns + * @throws Exception if running pushdown query fails + */ + void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta) throws Exception; +} http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java deleted file mode 100644 index cfb0f32..0000000 --- a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.source.adhocquery; - -import org.junit.Test; - -import junit.framework.TestCase; - -public class HiveAdhocConverterTest extends TestCase { - @Test - public void testStringReplace() { - String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10"; - String replacedString = HiveAdhocConverter.replaceString(originString, "char_length", "length"); - assertEquals("select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10", replacedString); - } - - @Test - public void testExtractReplace() { - String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore"; - String replacedString = HiveAdhocConverter.extractReplace(originString); - assertEquals("ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore", replacedString); - } - - @Test - public void testCastReplace() { - String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore"; - String replacedString = HiveAdhocConverter.castReplace(originString); - assertEquals("ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore", replacedString); - } - - @Test - public void testSubqueryReplace1() { - String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name"; - String replacedString = HiveAdhocConverter.subqueryReplace(originString); - assertEquals( - "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name", - replacedString); - } - - @Test - public void testSubqueryReplace2() { - String originString = "select count(*) from ( select test_kylin_fact.lstg_format_name from test_kylin_fact where test_kylin_fact.lstg_format_name='FP-GTC' group by test_kylin_fact.lstg_format_name ) t "; - String replacedString = HiveAdhocConverter.subqueryReplace(originString); - assertEquals(originString, replacedString); - } - - @Test - public void testSubqueryReplace3() { - String originString = "select fact.lstg_format_name from (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact group by fact.lstg_format_name order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE fact.lstg_format_name END "; - String replacedString = HiveAdhocConverter.subqueryReplace(originString); - assertEquals(originString, replacedString); - } - - @Test - public void testConcatReplace() { - String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'"; - String replacedString = HiveAdhocConverter.concatReplace(originString); - assertEquals("select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'", - replacedString); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java new file mode 100644 index 0000000..42c628a --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HivePushDownConverterTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.source.adhocquery; + +import org.junit.Test; + +import junit.framework.TestCase; + +public class HivePushDownConverterTest extends TestCase { + @Test + public void testStringReplace() { + String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10"; + String replacedString = HivePushDownConverter.replaceString(originString, "char_length", "length"); + assertEquals("select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10", replacedString); + } + + @Test + public void testExtractReplace() { + String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore"; + String replacedString = HivePushDownConverter.extractReplace(originString); + assertEquals("ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore", replacedString); + } + + @Test + public void testCastReplace() { + String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore"; + String replacedString = HivePushDownConverter.castReplace(originString); + assertEquals("ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore", replacedString); + } + + @Test + public void testSubqueryReplace1() { + String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name"; + String replacedString = HivePushDownConverter.subqueryReplace(originString); + assertEquals( + "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name", + replacedString); + } + + @Test + public void testSubqueryReplace2() { + String originString = "select count(*) from ( select test_kylin_fact.lstg_format_name from test_kylin_fact where test_kylin_fact.lstg_format_name='FP-GTC' group by test_kylin_fact.lstg_format_name ) t "; + String replacedString = HivePushDownConverter.subqueryReplace(originString); + assertEquals(originString, replacedString); + } + + @Test + public void testSubqueryReplace3() { + String originString = "select fact.lstg_format_name from (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact group by fact.lstg_format_name order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE fact.lstg_format_name END "; + String replacedString = HivePushDownConverter.subqueryReplace(originString); + assertEquals(originString, replacedString); + } + + @Test + public void testConcatReplace() { + String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'"; + String replacedString = HivePushDownConverter.concatReplace(originString); + assertEquals("select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'", + replacedString); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 35386de..6a571df 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -186,14 +186,14 @@ kylin.engine.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -### AD-HOC QUERY ### -#kylin.query.ad-hoc.runner-class-name=org.apache.kylin.query.adhoc.AdHocRunnerJdbcImpl +### QUERY PUSH DOWN ### +#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl -#kylin.query.ad-hoc.jdbc.url=jdbc:hive2://sandbox:10000/default -#kylin.query.ad-hoc.jdbc.driver=org.apache.hive.jdbc.HiveDriver -#kylin.query.ad-hoc.jdbc.username=hive -#kylin.query.ad-hoc.jdbc.password= +#kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:10000/default +#kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver +#kylin.query.pushdown.jdbc.username=hive +#kylin.query.pushdown.jdbc.password= -#kylin.query.ad-hoc.jdbc.pool-max-total=8 -#kylin.query.ad-hoc.jdbc.pool-max-idle=8 -#kylin.query.ad-hoc.jdbc.pool-min-idle=0 +#kylin.query.pushdown.jdbc.pool-max-total=8 +#kylin.query.pushdown.jdbc.pool-max-idle=8 +#kylin.query.pushdown.jdbc.pool-min-idle=0 http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index af05237..3634100 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -51,7 +51,7 @@ import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; -import org.apache.kylin.rest.util.AdHocUtil; +import org.apache.kylin.rest.util.PushDownUtil; import org.dbunit.DatabaseUnitException; import org.dbunit.database.DatabaseConfig; import org.dbunit.database.DatabaseConnection; @@ -263,7 +263,7 @@ public class KylinTestBase { } catch (SQLException sqlException) { List<List<String>> results = Lists.newArrayList(); List<SelectedColumnMeta> columnMetas = Lists.newArrayList(); - AdHocUtil.doAdHocQuery(ProjectInstance.DEFAULT_PROJECT_NAME, sql, results, columnMetas, sqlException); + PushDownUtil.doPushDownQuery(ProjectInstance.DEFAULT_PROJECT_NAME, sql, results, columnMetas, sqlException); return results.size(); } finally { if (resultSet != null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java deleted file mode 100644 index 852cebf..0000000 --- a/query/src/main/java/org/apache/kylin/query/adhoc/AdHocRunnerJdbcImpl.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.query.adhoc; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.LinkedList; -import java.util.List; - -import org.apache.commons.pool.impl.GenericObjectPool; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; -import org.apache.kylin.source.adhocquery.IAdHocRunner; - -public class AdHocRunnerJdbcImpl implements IAdHocRunner { - - private static org.apache.kylin.query.adhoc.JdbcConnectionPool pool = null; - - @Override - public void init(KylinConfig config) { - if (pool == null) { - pool = new JdbcConnectionPool(); - JdbcConnectionFactory factory = new JdbcConnectionFactory(config.getJdbcUrl(), config.getJdbcDriverClass(), - config.getJdbcUsername(), config.getJdbcPassword()); - GenericObjectPool.Config poolConfig = new GenericObjectPool.Config(); - poolConfig.maxActive = config.getPoolMaxTotal(); - poolConfig.maxIdle = config.getPoolMaxIdle(); - poolConfig.minIdle = config.getPoolMinIdle(); - - try { - pool.createPool(factory, poolConfig); - } catch (IOException e) { - throw new RuntimeException(e.getMessage(), e); - } - } - } - - @Override - public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas) - throws Exception { - Statement statement = null; - Connection connection = this.getConnection(); - ResultSet resultSet = null; - - try { - statement = connection.createStatement(); - resultSet = statement.executeQuery(query); - extractResults(resultSet, results); - } catch (SQLException sqlException) { - throw sqlException; - } - - //extract column metadata - ResultSetMetaData metaData = null; - int columnCount = 0; - try { - metaData = resultSet.getMetaData(); - columnCount = metaData.getColumnCount(); - - // fill in selected column meta - for (int i = 1; i <= columnCount; ++i) { - columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, - metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), - metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, - metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), - metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false)); - } - - } catch (SQLException sqlException) { - throw sqlException; - } - - closeConnection(connection); - } - - private Connection getConnection() { - return pool.getConnection(); - } - - private void closeConnection(Connection connection) { - pool.returnConnection(connection); - } - - static void extractResults(ResultSet resultSet, List<List<String>> results) throws SQLException { - List<String> oneRow = new LinkedList<String>(); - - try { - while (resultSet.next()) { - //logger.debug("resultSet value: " + resultSet.getString(1)); - for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) { - oneRow.add((resultSet.getString(i + 1))); - } - - results.add(new LinkedList<String>(oneRow)); - oneRow.clear(); - } - } catch (SQLException sqlException) { - throw sqlException; - } - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java new file mode 100644 index 0000000..8001880 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.adhoc; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.pool.impl.GenericObjectPool; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; +import org.apache.kylin.source.adhocquery.IPushDownRunner; + +public class PushDownRunnerJdbcImpl implements IPushDownRunner { + + private static org.apache.kylin.query.adhoc.JdbcConnectionPool pool = null; + + @Override + public void init(KylinConfig config) { + if (pool == null) { + pool = new JdbcConnectionPool(); + JdbcConnectionFactory factory = new JdbcConnectionFactory(config.getJdbcUrl(), config.getJdbcDriverClass(), + config.getJdbcUsername(), config.getJdbcPassword()); + GenericObjectPool.Config poolConfig = new GenericObjectPool.Config(); + poolConfig.maxActive = config.getPoolMaxTotal(); + poolConfig.maxIdle = config.getPoolMaxIdle(); + poolConfig.minIdle = config.getPoolMinIdle(); + + try { + pool.createPool(factory, poolConfig); + } catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + } + + @Override + public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas) + throws Exception { + Statement statement = null; + Connection connection = this.getConnection(); + ResultSet resultSet = null; + + try { + statement = connection.createStatement(); + resultSet = statement.executeQuery(query); + extractResults(resultSet, results); + } catch (SQLException sqlException) { + throw sqlException; + } + + //extract column metadata + ResultSetMetaData metaData = null; + int columnCount = 0; + try { + metaData = resultSet.getMetaData(); + columnCount = metaData.getColumnCount(); + + // fill in selected column meta + for (int i = 1; i <= columnCount; ++i) { + columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, + metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), + metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, + metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), + metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false)); + } + + } catch (SQLException sqlException) { + throw sqlException; + } + + closeConnection(connection); + } + + private Connection getConnection() { + return pool.getConnection(); + } + + private void closeConnection(Connection connection) { + pool.returnConnection(connection); + } + + static void extractResults(ResultSet resultSet, List<List<String>> results) throws SQLException { + List<String> oneRow = new LinkedList<String>(); + + try { + while (resultSet.next()) { + //logger.debug("resultSet value: " + resultSet.getString(1)); + for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) { + oneRow.add((resultSet.getString(i + 1))); + } + + results.add(new LinkedList<String>(oneRow)); + oneRow.clear(); + } + } catch (SQLException sqlException) { + throw sqlException; + } + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/7002dd86/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java index d841dee..79a2c05 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java +++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java @@ -61,12 +61,13 @@ public class SQLResponse implements Serializable { protected boolean storageCacheUsed = false; - protected boolean queryAdHoc = false; + protected boolean queryPushDown = false; public SQLResponse() { } - public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, int affectedRowCount, boolean isException, String exceptionMessage) { + public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, int affectedRowCount, + boolean isException, String exceptionMessage) { this.columnMetas = columnMetas; this.results = results; this.affectedRowCount = affectedRowCount; @@ -74,7 +75,8 @@ public class SQLResponse implements Serializable { this.exceptionMessage = exceptionMessage; } - public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube, int affectedRowCount, boolean isException, String exceptionMessage) { + public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube, + int affectedRowCount, boolean isException, String exceptionMessage) { this.columnMetas = columnMetas; this.results = results; this.cube = cube; @@ -83,7 +85,8 @@ public class SQLResponse implements Serializable { this.exceptionMessage = exceptionMessage; } - public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube, int affectedRowCount, boolean isException, String exceptionMessage, boolean isPartial, boolean isAdhoc) { + public SQLResponse(List<SelectedColumnMeta> columnMetas, List<List<String>> results, String cube, + int affectedRowCount, boolean isException, String exceptionMessage, boolean isPartial, boolean isPushDown) { this.columnMetas = columnMetas; this.results = results; this.cube = cube; @@ -91,7 +94,7 @@ public class SQLResponse implements Serializable { this.isException = isException; this.exceptionMessage = exceptionMessage; this.isPartial = isPartial; - this.queryAdHoc = isAdhoc; + this.queryPushDown = isPushDown; } public List<SelectedColumnMeta> getColumnMetas() { @@ -147,8 +150,8 @@ public class SQLResponse implements Serializable { return isPartial; } - public boolean isAdHoc() { - return queryAdHoc; + public boolean isPushDown() { + return queryPushDown; } public long getTotalScanCount() {
