KYLIN-2540 fix concat cascade issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3296f6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3296f6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3296f6 Branch: refs/heads/yang22-hbase1.x Commit: 1c3296f65a4939df57502e38f115f1516c27ef0e Parents: 9d4d741 Author: lidongsjtu <lid...@apache.org> Authored: Mon Apr 10 21:50:32 2017 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Mon Apr 10 21:54:11 2017 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 1 + .../test_case_data/sandbox/kylin.properties | 1 + .../src/test/resources/query/sql/query104.sql | 31 ++++++++++++++++++++ .../org/apache/kylin/query/udf/ConcatUDF.java | 28 ++++++++++++++++++ .../org/apache/kylin/rest/util/QueryUtil.java | 15 ---------- 5 files changed, 61 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 7b93c94..d00d714 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -149,6 +149,7 @@ kylin.snapshot.max-mb=300 kylin.query.scan-threshold=10000000 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF +kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF # 3G kylin.query.memory-budget-bytes=3221225472 http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index a77a061..b4e32d6 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -104,6 +104,7 @@ kylin.storage.hbase.max-region-count=5 kylin.storage.hbase.hfile-size-gb=2 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF +kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF # for test http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/kylin-it/src/test/resources/query/sql/query104.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query104.sql b/kylin-it/src/test/resources/query/sql/query104.sql new file mode 100644 index 0000000..bb85664 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query104.sql @@ -0,0 +1,31 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + + +select concat(meta_categ_name, lstg_format_name) as c1, concat(meta_categ_name, 'CONST') as c2, concat(meta_categ_name, concat(test_sites.site_name, lstg_format_name)) as c3, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name not in ('Crafts','Computers')) + group by concat(meta_categ_name, lstg_format_name), concat(meta_categ_name, 'CONST'), concat(meta_categ_name, concat(test_sites.site_name, lstg_format_name)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java new file mode 100644 index 0000000..9c0da53 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.query.udf; + +import org.apache.calcite.linq4j.function.Parameter; + +public class ConcatUDF { + + public String eval(@Parameter(name = "str1") String col1, @Parameter(name = "str2") String col2) { + return col1 + col2; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java index 59a4a78..26d6015 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java +++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java @@ -97,7 +97,6 @@ public class QueryUtil { private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE); private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!="+ S0, Pattern.CASE_INSENSITIVE); private static final Pattern PTN_INTERVAL = Pattern.compile("interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + "(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_CONCAT = Pattern.compile("concat\\(.+?\\)");//non-greedy private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE); @Override @@ -150,20 +149,6 @@ public class QueryUtil { sql = sql.substring(0, m.start(1)) + "'" + value + "'" + sql.substring(m.end(3)); } - //according to https://issues.apache.org/jira/browse/CALCITE-1375, - //{fn concat('a','b')} will succeed but concat('a','b') will fail - StringBuilder sb = new StringBuilder(); - while (true) { - m = PTN_CONCAT.matcher(sql); - if (!m.find()) - break; - - sb.append(sql.substring(0, m.start()) + "{fn " + m.group(0) + " }"); - sql = sql.substring(m.end()); - } - String temp = sb.toString() + sql; - sql = "".equals(temp) ? sql : temp; - return sql; }
