KYLIN-2539 Useless filter dimension will impact cuboid selection.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/177079d1 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/177079d1 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/177079d1 Branch: refs/heads/KYLIN-2506 Commit: 177079d10c1dfd4decadd743eafdfc36a712e470 Parents: c17b3eb Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Apr 10 21:21:41 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Mon Apr 10 21:22:09 2017 +0800 ---------------------------------------------------------------------- .../metadata/filter/CompareTupleFilter.java | 14 +++ .../filter/FilterOptimizeTransformer.java | 89 ++++++++++++++++++++ .../gtrecord/GTCubeStorageQueryBase.java | 6 +- 3 files changed, 108 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java index d567c89..d783e52 100755 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java @@ -218,6 +218,20 @@ public class CompareTupleFilter extends TupleFilter { && !conditionValues.isEmpty() && secondColumn == null; } + public boolean alwaysReturnTrue() { + // 1 = 1 + if (this.operator == FilterOperatorEnum.EQ) { + if (this.children != null && this.children.size() == 2 && // + this.children.get(0) instanceof ConstantTupleFilter && // + this.children.get(1) instanceof ConstantTupleFilter && // + ((ConstantTupleFilter) this.children.get(0)).getValues().equals(((ConstantTupleFilter) this.children.get(1)).getValues())) { + return true; + } + } + + return false; + } + @SuppressWarnings({ "unchecked", "rawtypes" }) @Override public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) { http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java new file mode 100755 index 0000000..a3bbad6 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter; + +import java.util.ListIterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * optimize the filter if possible, not limited to: + * <p> + * 1. prune filters like (a = ? OR 1 = 1) + */ +public class FilterOptimizeTransformer implements ITupleFilterTransformer { + public static final Logger logger = LoggerFactory.getLogger(FilterOptimizeTransformer.class); + + @Override + public TupleFilter transform(TupleFilter tupleFilter) { + TupleFilter translated = null; + if (tupleFilter instanceof CompareTupleFilter) { + //normal case + translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) tupleFilter); + } else if (tupleFilter instanceof LogicalTupleFilter) { + @SuppressWarnings("unchecked") + ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); + while (childIterator.hasNext()) { + TupleFilter transformed = transform(childIterator.next()); + if (transformed != null) { + childIterator.set(transformed); + } else { + throw new IllegalStateException("Should not be null"); + } + } + + translated = replaceAlwaysTrueLogicalFilter((LogicalTupleFilter) tupleFilter); + + } + return translated == null ? tupleFilter : translated; + + } + + private TupleFilter replaceAlwaysTrueLogicalFilter(LogicalTupleFilter logicalTupleFilter) { + if (logicalTupleFilter == null) { + return null; + } + + if (logicalTupleFilter.getOperator() == TupleFilter.FilterOperatorEnum.OR) { + @SuppressWarnings("unchecked") + ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) logicalTupleFilter.getChildren().listIterator(); + while (childIterator.hasNext()) { + TupleFilter next = childIterator.next(); + if (ConstantTupleFilter.TRUE == next) { + logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", logicalTupleFilter); + return ConstantTupleFilter.TRUE; + } + } + } + + return logicalTupleFilter; + } + + private TupleFilter replaceAlwaysTrueCompareFilter(CompareTupleFilter compareTupleFilter) { + + if (compareTupleFilter != null && compareTupleFilter.alwaysReturnTrue()) { + logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", compareTupleFilter); + return ConstantTupleFilter.TRUE; + } + + return compareTupleFilter; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 065fd6e..abac33a 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -38,6 +38,7 @@ import org.apache.kylin.dict.lookup.LookupStringTable; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; +import org.apache.kylin.metadata.filter.FilterOptimizeTransformer; import org.apache.kylin.metadata.filter.LogicalTupleFilter; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; @@ -148,13 +149,16 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { // set query deadline context.setDeadline(cubeInstance); + // optimize the filter, the optimization has to be segment-irrelevant + new FilterOptimizeTransformer().transform(filterD); + logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation()); return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, context); } protected abstract String getGTStorage(); - + protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) { return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo); }
