KYLIN-2301 various fixes to support NOT IN subquery
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0453d3da Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0453d3da Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0453d3da Branch: refs/heads/master-cdh5.7 Commit: 0453d3daada2e32f79954dfd12ac9f19062e825c Parents: ed8b168 Author: Li Yang <liy...@apache.org> Authored: Fri Dec 23 08:17:28 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Mon Dec 26 15:26:25 2016 +0800 ---------------------------------------------------------------------- .../kylin/metadata/filter/LogicalTupleFilter.java | 13 ------------- .../org/apache/kylin/metadata/filter/TupleFilter.java | 7 ++++++- .../org/apache/kylin/metadata/model/TblColRef.java | 7 ++----- .../java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +- .../src/test/resources/query/sql_subquery/query13.sql | 7 +++++++ .../apache/kylin/query/relnode/OLAPAggregateRel.java | 11 +++++++++-- .../org/apache/kylin/query/relnode/OLAPJoinRel.java | 8 +++++++- 7 files changed, 32 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java index 1744309..373acdb 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java @@ -47,19 +47,6 @@ public class LogicalTupleFilter extends TupleFilter { return cloneTuple; } - // private TupleFilter reverseNestedNots(TupleFilter filter, int depth) { - // if ((filter instanceof LogicalTupleFilter) && (filter.operator == FilterOperatorEnum.NOT)) { - // assert (filter.children.size() == 1); - // return reverseNestedNots(filter.children.get(0), depth + 1); - // } - // - // if (depth % 2 == 1) { - // return filter; - // } else { - // return filter.reverse(); - // } - // } - @Override public TupleFilter reverse() { switch (operator) { http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java index 285172c..31de50b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java @@ -27,6 +27,8 @@ import java.util.Set; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Maps; @@ -37,6 +39,8 @@ import com.google.common.collect.Maps; */ public abstract class TupleFilter { + static final Logger logger = LoggerFactory.getLogger(TupleFilter.class); + public enum FilterOperatorEnum { EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), EVAL_FUNC(37), UNSUPPORTED(38); @@ -112,7 +116,8 @@ public abstract class TupleFilter { } public TupleFilter reverse() { - throw new UnsupportedOperationException(); + logger.warn("Cannot reverse " + this + ", loosen the filter to true"); + return ConstantTupleFilter.TRUE; } /** http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java index fd0224f..88abb9d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java @@ -27,7 +27,7 @@ import org.apache.kylin.metadata.datatype.DataType; /** */ -@SuppressWarnings("serial") +@SuppressWarnings({ "serial", "deprecation" }) public class TblColRef implements Serializable { private static final String INNER_TABLE_NAME = "_kylin_table"; @@ -82,6 +82,7 @@ public class TblColRef implements Serializable { TableRef tableRef = model.findTable(alias); checkArgument(tableRef.getTableDesc() == col.column.getTable()); col.table = tableRef; + col.identity = null; } // for test mainly @@ -160,10 +161,6 @@ public class TblColRef implements Serializable { return InnerDataTypeEnum.contains(getDatatype()); } - public boolean isDerivedDataType() { - return InnerDataTypeEnum.DERIVED.getDataType().equals(getDatatype()); - } - public int hashCode() { // NOTE: tableRef MUST NOT participate in hashCode(). // Because fixUnknownModel() can change tableRef while TblColRef is held as set/map keys. http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 4329141..840a03f 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -149,7 +149,7 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testSingleRunQuery() throws Exception { - String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_limit/query01.sql"; + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/temp/sample.sql"; File sqlFile = new File(queryFileName); if (sqlFile.exists()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/kylin-it/src/test/resources/query/sql_subquery/query13.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query13.sql b/kylin-it/src/test/resources/query/sql_subquery/query13.sql new file mode 100644 index 0000000..d7f6109 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_subquery/query13.sql @@ -0,0 +1,7 @@ +select + f.cal_dt +from test_kylin_fact f +where + f.cal_dt not in ( + select cal_dt from EDW.TEST_CAL_DT where week_beg_dt = date'2012-01-01' + ) http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java index c457545..8d7c597 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java @@ -153,7 +153,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { // only translate the innermost aggregation if (!this.afterAggregate) { - this.context.groupByColumns.addAll(this.groups); + addToContextGroupBy(this.groups); this.context.aggregations.addAll(this.aggregations); this.context.afterAggregate = true; @@ -326,7 +326,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { FunctionDesc aggFunc = this.aggregations.get(i); if (aggFunc.isDimensionAsMetric()) { - this.context.groupByColumns.addAll(aggFunc.getParameter().getColRefs()); + addToContextGroupBy(aggFunc.getParameter().getColRefs()); continue; // skip rewrite, let calcite handle } @@ -350,6 +350,13 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { } } + private void addToContextGroupBy(List<TblColRef> colRefs) { + for (TblColRef col : colRefs) { + if (col.isInnerColumn() == false) + this.context.groupByColumns.add(col); + } + } + private void fillbackOptimizedColumn() { // some aggcall will be optimized out in sub-query (e.g. tableau generated sql), we need to fill them back RelDataType inputAggRow = getInput().getRowType(); http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java index 6869858..54b851f 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java @@ -164,7 +164,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { } else if (leftHasSubquery != rightHasSubquery) { //When join contains subquery, the join-condition fields of fact_table will add into context. Map<TblColRef, TblColRef> joinCol = new HashMap<TblColRef, TblColRef>(); - translateJoinColumn((RexCall) this.getCondition(), joinCol); + translateJoinColumn(this.getCondition(), joinCol); for (Map.Entry<TblColRef, TblColRef> columnPair : joinCol.entrySet()) { TblColRef fromCol = (rightHasSubquery ? columnPair.getKey() : columnPair.getValue()); @@ -219,6 +219,12 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { return join; } + private void translateJoinColumn(RexNode condition, Map<TblColRef, TblColRef> joinCol) { + if (condition instanceof RexCall) { + translateJoinColumn((RexCall) condition, joinCol); + } + } + private void translateJoinColumn(RexCall condition, Map<TblColRef, TblColRef> joinColumns) { SqlKind kind = condition.getOperator().getKind(); if (kind == SqlKind.AND) {
