Repository: kylin Updated Branches: refs/heads/master 0eebd7d8a -> eb686a942
KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result Signed-off-by: Li Yang <liy...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fd1ed6f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fd1ed6f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fd1ed6f Branch: refs/heads/master Commit: 0fd1ed6f14758bb38f1abf7a7ed9e24ed818f50f Parents: 0eebd7d Author: zhengdong <zhe...@outlook.com> Authored: Wed Nov 23 16:13:59 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri Dec 2 18:32:24 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/gridtable/GTUtil.java | 14 ++++++++++++ .../metadata/filter/CompareTupleFilter.java | 2 ++ .../metadata/filter/TupleFilterSerializer.java | 1 + .../resources/query/sql_derived/query12.sql | 24 ++++++++++++++++++++ .../kylin/query/relnode/OLAPFilterRel.java | 3 +++ .../common/coprocessor/FilterDecorator.java | 14 ++++++++++++ 6 files changed, 58 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java old mode 100644 new mode 100755 index a4e574c..7496778 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java @@ -196,6 +196,20 @@ public class GTUtil { result = newCompareFilter; } break; + case NOTIN: + Set notInValues = Sets.newHashSet(); + for (Object value : constValues) { + code = translate(col, value, 0); + if (code != null) + notInValues.add(code); + } + if (notInValues.isEmpty()) { + result = ConstantTupleFilter.TRUE; + } else { + newCompareFilter.addChild(new ConstantTupleFilter(notInValues)); + result = newCompareFilter; + } + break; case NEQ: code = translate(col, firstValue, 0); if (code == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java old mode 100644 new mode 100755 index c7a3721..f2af735 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java @@ -59,6 +59,8 @@ public class CompareTupleFilter extends TupleFilter { private CompareTupleFilter(CompareTupleFilter another) { super(new ArrayList<TupleFilter>(another.children), another.operator); this.column = another.column; + this.firstCondValue = another.getFirstValue(); + this.function = another.getFunction(); this.conditionValues = new HashSet<Object>(); this.conditionValues.addAll(another.conditionValues); this.dynamicVariables = new HashMap<String, Object>(); http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java old mode 100644 new mode 100755 index 2df474e..63153ef --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java @@ -170,6 +170,7 @@ public class TupleFilterSerializer { case GT: case GTE: case IN: + case NOTIN: case ISNULL: case ISNOTNULL: filter = new CompareTupleFilter(op); http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/kylin-it/src/test/resources/query/sql_derived/query12.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql b/kylin-it/src/test/resources/query/sql_derived/query12.sql new file mode 100755 index 0000000..959a59c --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_derived/query12.sql @@ -0,0 +1,24 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select UPD_USER,count(1) as CNT +from TEST_KYLIN_FACT as TEST_KYLIN_FACT +JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +where UPD_USER not in ('USER_Y') +group by UPD_USER \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java old mode 100644 new mode 100755 index 8b2035b..411142d --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -160,6 +160,9 @@ public class OLAPFilterRel extends Filter implements OLAPRel { if (inFilter != null) { filter = inFilter; } + } else if (op.getKind() == SqlKind.NOT) { + assert (filter.getChildren().size() == 1); + filter = filter.getChildren().get(0).reverse(); } return filter; } http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java old mode 100644 new mode 100755 index 5ab4117..2b2e490 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java @@ -90,6 +90,20 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator { result = newCompareFilter; } break; + case NOTIN: + Set<String> notInValues = Sets.newHashSet(); + for (String value : constValues) { + v = translate(col, value, 0); + if (!isDictNull(v)) + notInValues.add(v); + } + if (notInValues.isEmpty()) { + result = ConstantTupleFilter.TRUE; + } else { + newCompareFilter.addChild(new ConstantTupleFilter(notInValues)); + result = newCompareFilter; + } + break; case NEQ: v = translate(col, firstValue, 0); if (isDictNull(v)) {
