KYLIN-2598 Should not translate filter to a in-clause filter with too many elements
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9dc6553d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9dc6553d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9dc6553d Branch: refs/heads/master Commit: 9dc6553d29eb3f106823026f45dc878347a2a108 Parents: 529b1be Author: Hongbin Ma <mahong...@apache.org> Authored: Tue May 9 18:33:21 2017 +0800 Committer: Dong Li <lid...@apache.org> Committed: Tue May 9 20:30:21 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 4 +++ .../kylin/dict/BuiltInFunctionTransformer.java | 10 +++++- .../test/resources/query/sql_like/query01.sql | 6 ++-- .../test/resources/query/sql_like/query05.sql | 33 ++++++++++++++++++++ .../test/resources/query/sql_like/query06.sql | 33 ++++++++++++++++++++ .../test/resources/query/sql_like/query22.sql | 31 ++++++++++++++++++ .../test/resources/query/sql_like/query23.sql | 31 ++++++++++++++++++ 7 files changed, 144 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index eafdbbb..10209d7 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -844,6 +844,10 @@ abstract public class KylinConfigBase implements Serializable { return value > 0 ? value : Long.MAX_VALUE; } + public int getTranslatedInClauseMaxSize() { + return Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size", "1024")); + } + public int getLargeQueryThreshold() { return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf(1000000))); } http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java index 6ef2f96..4d7b15d 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java @@ -21,6 +21,7 @@ package org.apache.kylin.dict; import java.util.Collection; import java.util.ListIterator; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.dimension.IDimensionEncodingMap; import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter; @@ -93,12 +94,19 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer { translated.addChild(new ColumnTupleFilter(columnRef)); try { + int translatedInClauseMaxSize = KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize(); + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { Object dictVal = dict.getValueFromId(i); if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) { translated.addChild(new ConstantTupleFilter(dictVal)); + + if (translated.getChildren().size() > translatedInClauseMaxSize) { + return null; + } } } + } catch (Exception e) { logger.debug(e.getMessage()); return null; @@ -175,4 +183,4 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer { } return translated; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql b/kylin-it/src/test/resources/query/sql_like/query01.sql index e1f131d..e3a05ca 100644 --- a/kylin-it/src/test/resources/query/sql_like/query01.sql +++ b/kylin-it/src/test/resources/query/sql_like/query01.sql @@ -16,7 +16,7 @@ -- limitations under the License. -- -select lstg_format_name as lstg_format_name, count(*) as cnt +select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt @@ -27,5 +27,5 @@ inner JOIN edw.test_cal_dt as test_cal_dt ON test_kylin_fact.lstg_site_id = test_sites.site_id -where lstg_format_name like '%BIN%' -group by lstg_format_name \ No newline at end of file +where META_CATEG_NAME like '%ab%' +group by META_CATEG_NAME http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query05.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_like/query05.sql b/kylin-it/src/test/resources/query/sql_like/query05.sql new file mode 100644 index 0000000..6ac9758 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_like/query05.sql @@ -0,0 +1,33 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select upper(meta_categ_name) as META_CATEG_NAME, count(*) as cnt + + +from test_kylin_fact +inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + +where lower(meta_categ_name)='baby' and substring(meta_categ_name,1,3) in ('Bab') and upper(meta_categ_name) > 'AAAA' and +lower(meta_categ_name) like '%b%' and char_length(meta_categ_name) < 10 and char_length(meta_categ_name) > 3 and meta_categ_name||'a'='Babya' +group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query06.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_like/query06.sql b/kylin-it/src/test/resources/query/sql_like/query06.sql new file mode 100644 index 0000000..6417584 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_like/query06.sql @@ -0,0 +1,33 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- test case for KYLIN-1954 + +select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt + + from test_kylin_fact +inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + +where META_CATEG_NAME like '%ab%' and META_CATEG_NAME > 'A' +group by META_CATEG_NAME http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query22.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_like/query22.sql b/kylin-it/src/test/resources/query/sql_like/query22.sql new file mode 100644 index 0000000..a962d3f --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_like/query22.sql @@ -0,0 +1,31 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt + + from test_kylin_fact +inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + +where META_CATEG_NAME not like '%ab%' +group by META_CATEG_NAME http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query23.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_like/query23.sql b/kylin-it/src/test/resources/query/sql_like/query23.sql new file mode 100644 index 0000000..b3a78c2 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_like/query23.sql @@ -0,0 +1,31 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt + + from test_kylin_fact +inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + +where not(META_CATEG_NAME not like '%ab%') +group by META_CATEG_NAME
