minor, code refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5ee76e8b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5ee76e8b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5ee76e8b Branch: refs/heads/1.5.x-CDH5.7 Commit: 5ee76e8b5840e7f8c6da9a56e6f1f496f0821589 Parents: 57a259a Author: Hongbin Ma <mahong...@apache.org> Authored: Thu Aug 25 14:17:19 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Fri Aug 26 10:39:19 2016 +0800 ---------------------------------------------------------------------- .../kylin/cube/CubeCapabilityChecker.java | 50 +- .../org/apache/kylin/cube/CubeInstance.java | 2 +- .../java/org/apache/kylin/cube/CubeSegment.java | 10 +- .../java/org/apache/kylin/cube/ISegment.java | 39 ++ .../java/org/apache/kylin/cube/JoinChecker.java | 68 +++ .../gridtable/AsymmetricRecordComparator.java | 55 ++ .../kylin/cube/gridtable/ComparatorEx.java | 64 +++ .../kylin/cube/gridtable/CubeGridTable.java | 21 - .../cube/gridtable/CubeScanRangePlanner.java | 542 +------------------ .../gridtable/CuboidToGridTableMapping.java | 60 +- .../kylin/cube/gridtable/RecordComparator.java | 46 ++ .../kylin/cube/gridtable/RecordComparators.java | 77 +++ .../cube/gridtable/ScanRangePlannerBase.java | 295 ++++++++++ .../cube/gridtable/SegmentGTStartAndEnd.java | 84 +++ .../java/org/apache/kylin/gridtable/GTInfo.java | 16 +- .../apache/kylin/gridtable/GTScanRequest.java | 16 +- .../kylin/gridtable/GTScanRequestBuilder.java | 13 + .../apache/kylin/gridtable/ScannerWorker.java | 8 +- .../kylin/gridtable/DictGridTableTest.java | 2 +- .../kylin/metadata/project/ProjectL2Cache.java | 8 + .../kylin/metadata/realization/SQLDigest.java | 6 +- .../storage/gtrecord/CubeTupleConverter.java | 42 +- .../gtrecord/GTCubeStorageQueryBase.java | 2 +- .../gtrecord/SequentialCubeTupleIterator.java | 4 +- .../apache/kylin/query/ITKylinQueryTest.java | 4 +- pom.xml | 6 + .../kylin/query/enumerator/OLAPEnumerator.java | 9 +- .../apache/kylin/query/routing/Candidate.java | 1 + .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 6 +- .../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 +- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 6 +- 31 files changed, 919 insertions(+), 650 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index 1f16b1b..e0d8dd3 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -18,7 +18,6 @@ package org.apache.kylin.cube; -import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; @@ -26,18 +25,16 @@ import java.util.List; import java.util.Set; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.DimensionDesc; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.basic.BasicMeasureType; import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IStorageAware; -import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.CapabilityResult; -import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.apache.kylin.metadata.realization.SQLDigest; +import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +50,7 @@ public class CubeCapabilityChecker { result.capable = false; // match joins - boolean isJoinMatch = isJoinMatch(digest.joinDescs, cube); + boolean isJoinMatch = JoinChecker.isJoinMatch(digest.joinDescs, cube); if (!isJoinMatch) { logger.info("Exclude cube " + cube.getName() + " because unmatched joins"); return result; @@ -89,7 +86,15 @@ public class CubeCapabilityChecker { if (cube.getStorageType() == IStorageAware.ID_HBASE && MassInTupleFilter.containsMassInTupleFilter(digest.filter)) { logger.info("Exclude cube " + cube.getName() + " because only v2 storage + v2 query engine supports massin"); return result; + } + if (digest.isRawQuery() && cube.getFactTable().equals(digest.factTable)) { + result.influences.add(new CapabilityInfluence() { + @Override + public double suggestCostMultiplier() { + return 100; + } + }); } // cost will be minded by caller @@ -121,41 +126,6 @@ public class CubeCapabilityChecker { return result; } - private static boolean isJoinMatch(Collection<JoinDesc> joins, CubeInstance cube) { - CubeDesc cubeDesc = cube.getDescriptor(); - - List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size()); - for (DimensionDesc d : cubeDesc.getDimensions()) { - if (d.getJoin() != null) { - cubeJoins.add(d.getJoin()); - } - } - for (JoinDesc j : joins) { - // optiq engine can't decide which one is fk or pk - String pTable = j.getPrimaryKeyColumns()[0].getTable(); - String factTable = cubeDesc.getFactTable(); - if (factTable.equals(pTable)) { - j.swapPKFK(); - } - - // check primary key, all PK column should refer to same tale, the Fact Table of cube. - // Using first column's table name to check. - String fTable = j.getForeignKeyColumns()[0].getTable(); - if (!factTable.equals(fTable)) { - logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + cube.getName()); - return false; - } - - // The hashcode() function of JoinDesc has been overwritten, - // which takes into consideration: pk,fk,jointype - if (!cubeJoins.contains(j)) { - logger.info("Query joins don't macth on cube " + cube.getName()); - return false; - } - } - return true; - } - private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) { CubeDesc cubeDesc = cube.getDescriptor(); Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions(); http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 90da59d..151e142 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -352,7 +352,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return result; } - private int getCost(SQLDigest digest) { + public int getCost(SQLDigest digest) { int calculatedCost = cost; //the number of dimensions is not as accurate as number of row key cols http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java index aaa88f1..79397c3 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java @@ -35,21 +35,22 @@ import org.apache.kylin.common.util.ShardingHash; import org.apache.kylin.cube.kv.CubeDimEncMap; import org.apache.kylin.cube.kv.RowConstants; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IBuildable; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.IRealization; import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class CubeSegment implements Comparable<CubeSegment>, IBuildable { +public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment { @JsonBackReference private CubeInstance cubeInstance; @@ -175,6 +176,11 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable { return status; } + @Override + public DataModelDesc getModel() { + return this.getCubeDesc().getModel(); + } + public void setStatus(SegmentStatusEnum status) { this.status = status; } http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java new file mode 100644 index 0000000..2e1f214 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube; + +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.SegmentStatusEnum; + +public interface ISegment { + + public String getName(); + + public long getDateRangeStart(); + + public long getDateRangeEnd(); + + public long getSourceOffsetStart(); + + public long getSourceOffsetEnd(); + + public DataModelDesc getModel(); + + public SegmentStatusEnum getStatus(); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java new file mode 100644 index 0000000..edd5be9 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube; + +import java.util.Collection; +import java.util.List; + +import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.LookupDesc; +import org.apache.kylin.metadata.realization.IRealization; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class JoinChecker { + + private static final Logger logger = LoggerFactory.getLogger(CubeCapabilityChecker.class); + + public static boolean isJoinMatch(Collection<JoinDesc> joins, IRealization realization) { + + List<JoinDesc> realizationsJoins = Lists.newArrayList(); + for (LookupDesc lookupDesc : realization.getDataModelDesc().getLookups()) { + realizationsJoins.add(lookupDesc.getJoin()); + } + + for (JoinDesc j : joins) { + // optiq engine can't decide which one is fk or pk + String pTable = j.getPrimaryKeyColumns()[0].getTable(); + String factTable = realization.getFactTable(); + if (factTable.equals(pTable)) { + j.swapPKFK(); + } + + // check primary key, all PK column should refer to same tale, the Fact Table of cube. + // Using first column's table name to check. + String fTable = j.getForeignKeyColumns()[0].getTable(); + if (!factTable.equals(fTable)) { + logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + realization.getName()); + return false; + } + + // The hashcode() function of JoinDesc has been overwritten, + // which takes into consideration: pk,fk,jointype + if (!realizationsJoins.contains(j)) { + logger.info("Query joins don't macth on cube " + realization.getName()); + return false; + } + } + return true; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java new file mode 100644 index 0000000..74dc855 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.gridtable; + +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.gridtable.GTRecord; + +import java.util.Collection; + +/** + * asymmetric means compare(a,b) > 0 does not cause compare(b,a) < 0 + * so min max functions will not be supported + */ +public class AsymmetricRecordComparator extends RecordComparator { + + AsymmetricRecordComparator(ComparatorEx<ByteArray> byteComparator) { + super(byteComparator); + } + + public GTRecord min(Collection<GTRecord> v) { + throw new UnsupportedOperationException(); + } + + public GTRecord max(Collection<GTRecord> v) { + throw new UnsupportedOperationException(); + } + + public GTRecord min(GTRecord a, GTRecord b) { + throw new UnsupportedOperationException(); + } + + public GTRecord max(GTRecord a, GTRecord b) { + throw new UnsupportedOperationException(); + } + + public boolean between(GTRecord v, GTRecord start, GTRecord end) { + throw new UnsupportedOperationException(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java new file mode 100644 index 0000000..3f1e8af --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.gridtable; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; + +public abstract class ComparatorEx<T> implements Comparator<T> { + + public T min(Collection<T> v) { + if (v.size() <= 0) { + return null; + } + + Iterator<T> iterator = v.iterator(); + T min = iterator.next(); + while (iterator.hasNext()) { + min = min(min, iterator.next()); + } + return min; + } + + public T max(Collection<T> v) { + if (v.size() <= 0) { + return null; + } + + Iterator<T> iterator = v.iterator(); + T max = iterator.next(); + while (iterator.hasNext()) { + max = max(max, iterator.next()); + } + return max; + } + + public T min(T a, T b) { + return compare(a, b) <= 0 ? a : b; + } + + public T max(T a, T b) { + return compare(a, b) >= 0 ? a : b; + } + + public boolean between(T v, T start, T end) { + return compare(start, v) <= 0 && compare(v, end) <= 0; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java index f2f3a25..563cf43 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java @@ -18,11 +18,9 @@ package org.apache.kylin.cube.gridtable; -import java.util.List; import java.util.Map; import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.kv.CubeDimEncMap; @@ -31,27 +29,8 @@ import org.apache.kylin.dimension.IDimensionEncodingMap; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.metadata.model.TblColRef; -import com.google.common.collect.Maps; - public class CubeGridTable { - public static Map<TblColRef, Dictionary<String>> getDimensionToDictionaryMap(CubeSegment cubeSeg, long cuboidId) { - CubeDesc cubeDesc = cubeSeg.getCubeDesc(); - CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig()); - - // build a dictionary map - Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap(); - List<TblColRef> dimCols = Cuboid.findById(cubeDesc, cuboidId).getColumns(); - for (TblColRef col : dimCols) { - Dictionary<String> dictionary = cubeMgr.getDictionary(cubeSeg, col); - if (dictionary != null) { - dictionaryMap.put(col, dictionary); - } - } - - return dictionaryMap; - } - public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) { Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId); return newGTInfo(cuboid, new CubeDimEncMap(cubeSeg)); http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java index 58771da..a937045 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java @@ -18,25 +18,17 @@ package org.apache.kylin.cube.gridtable; -import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.BitSet; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.common.FuzzyValueCombination; @@ -49,12 +41,7 @@ import org.apache.kylin.gridtable.GTScanRequest; import org.apache.kylin.gridtable.GTScanRequestBuilder; import org.apache.kylin.gridtable.GTUtil; import org.apache.kylin.gridtable.IGTComparator; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; @@ -64,7 +51,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -public class CubeScanRangePlanner { +public class CubeScanRangePlanner extends ScanRangePlannerBase { private static final Logger logger = LoggerFactory.getLogger(CubeScanRangePlanner.class); @@ -75,24 +62,6 @@ public class CubeScanRangePlanner { protected CubeSegment cubeSegment; protected CubeDesc cubeDesc; protected Cuboid cuboid; - protected TupleFilter filter; - protected Set<TblColRef> dimensions; - protected Set<TblColRef> groupbyDims; - protected Set<TblColRef> filterDims; - protected Collection<FunctionDesc> metrics; - - //GT - protected TupleFilter gtFilter; - protected GTInfo gtInfo; - protected Pair<ByteArray, ByteArray> gtStartAndEnd; - protected TblColRef gtPartitionCol; - protected ImmutableBitSet gtDimensions; - protected ImmutableBitSet gtAggrGroups; - protected ImmutableBitSet gtAggrMetrics; - protected String[] gtAggrFuncs; - final protected RecordComparator rangeStartComparator; - final protected RecordComparator rangeEndComparator; - final protected RecordComparator rangeStartEndComparator; public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupbyDims, // Collection<FunctionDesc> metrics) { @@ -103,36 +72,35 @@ public class CubeScanRangePlanner { this.cubeSegment = cubeSegment; this.cubeDesc = cubeSegment.getCubeDesc(); this.cuboid = cuboid; - this.dimensions = dimensions; - this.groupbyDims = groupbyDims; - this.filter = filter; - this.metrics = metrics; - this.filterDims = Sets.newHashSet(); - TupleFilter.collectColumns(filter, this.filterDims); + + Set<TblColRef> filterDims = Sets.newHashSet(); + TupleFilter.collectColumns(filter, filterDims); this.gtInfo = CubeGridTable.newGTInfo(cubeSegment, cuboid.getId()); CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping(); IGTComparator comp = gtInfo.getCodeSystem().getComparator(); //start key GTRecord compare to start key GTRecord - this.rangeStartComparator = getRangeStartComparator(comp); + this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp); //stop key GTRecord compare to stop key GTRecord - this.rangeEndComparator = getRangeEndComparator(comp); + this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp); //start key GTRecord compare to stop key GTRecord - this.rangeStartEndComparator = getRangeStartEndComparator(comp); + this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp); //replace the constant values in filter to dictionary codes - this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), this.groupbyDims); + this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupbyDims); - this.gtDimensions = makeGridTableColumns(mapping, dimensions); - this.gtAggrGroups = makeGridTableColumns(mapping, replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc())); - this.gtAggrMetrics = makeGridTableColumns(mapping, metrics); - this.gtAggrFuncs = makeAggrFuncs(mapping, metrics); + this.gtDimensions = mapping.makeGridTableColumns(dimensions); + this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc())); + this.gtAggrMetrics = mapping.makeGridTableColumns(metrics); + this.gtAggrFuncs = mapping.makeAggrFuncs(metrics); - if (cubeSegment.getCubeDesc().getModel().getPartitionDesc().isPartitioned()) { - int index = mapping.getIndexOf(cubeSegment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef()); + if (cubeSegment.getModel().getPartitionDesc().isPartitioned()) { + int index = mapping.getIndexOf(cubeSegment.getModel().getPartitionDesc().getPartitionDateColumnRef()); if (index >= 0) { - this.gtStartAndEnd = getSegmentStartAndEnd(index); + SegmentGTStartAndEnd segmentGTStartAndEnd = new SegmentGTStartAndEnd(cubeSegment, gtInfo); + this.gtStartAndEnd = segmentGTStartAndEnd.getSegmentStartAndEnd(index); + this.isPartitionColUsingDatetimeEncoding = segmentGTStartAndEnd.isUsingDatetimeEncoding(index); this.gtPartitionCol = gtInfo.colRef(index); } } @@ -155,11 +123,11 @@ public class CubeScanRangePlanner { IGTComparator comp = gtInfo.getCodeSystem().getComparator(); //start key GTRecord compare to start key GTRecord - this.rangeStartComparator = getRangeStartComparator(comp); + this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp); //stop key GTRecord compare to stop key GTRecord - this.rangeEndComparator = getRangeEndComparator(comp); + this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp); //start key GTRecord compare to stop key GTRecord - this.rangeStartEndComparator = getRangeStartEndComparator(comp); + this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp); this.gtFilter = gtFilter; this.gtStartAndEnd = gtStartAndEnd; @@ -199,46 +167,6 @@ public class CubeScanRangePlanner { return mergedRanges; } - private Pair<ByteArray, ByteArray> getSegmentStartAndEnd(int index) { - ByteArray start; - if (cubeSegment.getDateRangeStart() != Long.MIN_VALUE) { - start = encodeTime(cubeSegment.getDateRangeStart(), index, 1); - } else { - start = new ByteArray(); - } - - ByteArray end; - if (cubeSegment.getDateRangeEnd() != Long.MAX_VALUE) { - end = encodeTime(cubeSegment.getDateRangeEnd(), index, -1); - } else { - end = new ByteArray(); - } - return Pair.newPair(start, end); - - } - - private ByteArray encodeTime(long ts, int index, int roundingFlag) { - String value; - DataType partitionColType = gtInfo.getColumnType(index); - if (partitionColType.isDate()) { - value = DateFormat.formatToDateStr(ts); - } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) { - value = DateFormat.formatToTimeWithoutMilliStr(ts); - } else if (partitionColType.isStringFamily()) { - String partitionDateFormat = cubeSegment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat(); - if (StringUtils.isEmpty(partitionDateFormat)) - partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN; - value = DateFormat.formatToDateStr(ts, partitionDateFormat); - } else { - throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); - } - - ByteBuffer buffer = ByteBuffer.allocate(gtInfo.getMaxColumnLength()); - gtInfo.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer); - - return ByteArray.copyOf(buffer.array(), 0, buffer.position()); - } - private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) { Set<TblColRef> ret = Sets.newHashSet(); for (TblColRef col : input) { @@ -253,57 +181,6 @@ public class CubeScanRangePlanner { return ret; } - private ImmutableBitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Set<TblColRef> dimensions) { - BitSet result = new BitSet(); - for (TblColRef dim : dimensions) { - int idx = mapping.getIndexOf(dim); - if (idx >= 0) - result.set(idx); - } - return new ImmutableBitSet(result); - } - - private ImmutableBitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Collection<FunctionDesc> metrics) { - BitSet result = new BitSet(); - for (FunctionDesc metric : metrics) { - int idx = mapping.getIndexOf(metric); - if (idx < 0) - throw new IllegalStateException(metric + " not found in " + mapping); - result.set(idx); - } - return new ImmutableBitSet(result); - } - - private String[] makeAggrFuncs(final CuboidToGridTableMapping mapping, Collection<FunctionDesc> metrics) { - - //metrics are represented in ImmutableBitSet, which loses order information - //sort the aggrFuns to align with metrics natural order - List<FunctionDesc> metricList = Lists.newArrayList(metrics); - Collections.sort(metricList, new Comparator<FunctionDesc>() { - @Override - public int compare(FunctionDesc o1, FunctionDesc o2) { - int a = mapping.getIndexOf(o1); - int b = mapping.getIndexOf(o2); - return a - b; - } - }); - - String[] result = new String[metricList.size()]; - int i = 0; - for (FunctionDesc metric : metricList) { - result[i++] = metric.getExpression(); - } - return result; - } - - private String makeReadable(ByteArray byteArray) { - if (byteArray == null) { - return null; - } else { - return byteArray.toReadableText(); - } - } - protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) { GTRecord pkStart = new GTRecord(gtInfo); GTRecord pkEnd = new GTRecord(gtInfo); @@ -313,14 +190,14 @@ public class CubeScanRangePlanner { for (ColumnRange range : andDimRanges) { if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) { - if (rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end) <= 0 // - && (rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond()) < 0 // - || rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond()) == 0 // - && (range.op == FilterOperatorEnum.EQ || range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN))) { - //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <= when has equals in condition. + int beginCompare = rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond()); + int endCompare = rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end); + + if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) { + //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded when using dict encoding, so use <= when has equals in condition. } else { logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", // - new Object[] { gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end) }); + gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end)); return null; } } @@ -337,7 +214,9 @@ public class CubeScanRangePlanner { } } - fuzzyKeys = buildFuzzyKeys(fuzzyValues); + fuzzyKeys = + + buildFuzzyKeys(fuzzyValues); return new GTScanRange(pkStart, pkEnd, fuzzyKeys); } @@ -371,104 +250,6 @@ public class CubeScanRangePlanner { return result; } - protected TupleFilter flattenToOrAndFilter(TupleFilter filter) { - if (filter == null) - return null; - - TupleFilter flatFilter = filter.flatFilter(); - - // normalize to OR-AND filter - if (flatFilter.getOperator() == FilterOperatorEnum.AND) { - LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR); - f.addChild(flatFilter); - flatFilter = f; - } - - if (flatFilter.getOperator() != FilterOperatorEnum.OR) - throw new IllegalStateException(); - - return flatFilter; - } - - protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) { - List<Collection<ColumnRange>> result = Lists.newArrayList(); - - if (flatFilter == null) { - result.add(Collections.<ColumnRange> emptyList()); - return result; - } - - for (TupleFilter andFilter : flatFilter.getChildren()) { - if (andFilter.getOperator() != FilterOperatorEnum.AND) - throw new IllegalStateException("Filter should be AND instead of " + andFilter); - - Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren()); - if (andRanges != null) { - result.add(andRanges); - } - } - - return preEvaluateConstantConditions(result); - } - - private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) { - Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>(); - for (TupleFilter filter : andFilters) { - if ((filter instanceof CompareTupleFilter) == false) { - if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) { - return null; - } else { - continue; - } - } - - CompareTupleFilter comp = (CompareTupleFilter) filter; - if (comp.getColumn() == null) { - continue; - } - - @SuppressWarnings("unchecked") - ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(), comp.getOperator()); - ColumnRange existing = rangeMap.get(newRange.column); - if (existing == null) { - rangeMap.put(newRange.column, newRange); - } else { - existing.andMerge(newRange); - } - } - return rangeMap.values(); - } - - private List<Collection<ColumnRange>> preEvaluateConstantConditions(List<Collection<ColumnRange>> orAndRanges) { - boolean globalAlwaysTrue = false; - Iterator<Collection<ColumnRange>> iterator = orAndRanges.iterator(); - while (iterator.hasNext()) { - Collection<ColumnRange> andRanges = iterator.next(); - Iterator<ColumnRange> iterator2 = andRanges.iterator(); - boolean hasAlwaysFalse = false; - while (iterator2.hasNext()) { - ColumnRange range = iterator2.next(); - if (range.satisfyAll()) - iterator2.remove(); - else if (range.satisfyNone()) - hasAlwaysFalse = true; - } - if (hasAlwaysFalse) { - iterator.remove(); - } else if (andRanges.isEmpty()) { - globalAlwaysTrue = true; - break; - } - } - // return empty OR list means global false - // return an empty AND collection inside OR list means global true - if (globalAlwaysTrue) { - orAndRanges.clear(); - orAndRanges.add(Collections.<ColumnRange> emptyList()); - } - return orAndRanges; - } - protected List<GTScanRange> mergeOverlapRanges(List<GTScanRange> ranges) { if (ranges.size() <= 1) { return ranges; @@ -556,269 +337,4 @@ public class CubeScanRangePlanner { this.maxScanRanges = maxScanRanges; } - protected class ColumnRange { - private TblColRef column; - private ByteArray begin = ByteArray.EMPTY; - private ByteArray end = ByteArray.EMPTY; - private Set<ByteArray> valueSet; - private FilterOperatorEnum op; - - public ColumnRange(TblColRef column, Set<ByteArray> values, FilterOperatorEnum op) { - this.column = column; - this.op = op; - - switch (op) { - case EQ: - case IN: - valueSet = new HashSet<ByteArray>(values); - refreshBeginEndFromEquals(); - break; - case LT: - case LTE: - end = rangeEndComparator.comparator.max(values); - break; - case GT: - case GTE: - begin = rangeStartComparator.comparator.min(values); - break; - case NEQ: - case NOTIN: - case ISNULL: - case ISNOTNULL: - // let Optiq filter it! - break; - default: - throw new UnsupportedOperationException(op.name()); - } - } - - void copy(TblColRef column, ByteArray beginValue, ByteArray endValue, Set<ByteArray> equalValues) { - this.column = column; - this.begin = beginValue; - this.end = endValue; - this.valueSet = equalValues; - } - - private void refreshBeginEndFromEquals() { - if (valueSet.isEmpty()) { - begin = ByteArray.EMPTY; - end = ByteArray.EMPTY; - } else { - begin = rangeStartComparator.comparator.min(valueSet); - end = rangeEndComparator.comparator.max(valueSet); - } - } - - public boolean satisfyAll() { - return begin.array() == null && end.array() == null; // the NEQ case - } - - public boolean satisfyNone() { - if (valueSet != null) { - return valueSet.isEmpty(); - } else if (begin.array() != null && end.array() != null) { - return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0; - } else { - return false; - } - } - - public void andMerge(ColumnRange another) { - assert this.column.equals(another.column); - - if (another.satisfyAll()) { - return; - } - - if (this.satisfyAll()) { - copy(another.column, another.begin, another.end, another.valueSet); - return; - } - - if (this.valueSet != null && another.valueSet != null) { - this.valueSet.retainAll(another.valueSet); - refreshBeginEndFromEquals(); - return; - } - - if (this.valueSet != null) { - this.valueSet = filter(this.valueSet, another.begin, another.end); - refreshBeginEndFromEquals(); - return; - } - - if (another.valueSet != null) { - this.valueSet = filter(another.valueSet, this.begin, this.end); - refreshBeginEndFromEquals(); - return; - } - - this.begin = rangeStartComparator.comparator.max(this.begin, another.begin); - this.end = rangeEndComparator.comparator.min(this.end, another.end); - } - - private Set<ByteArray> filter(Set<ByteArray> equalValues, ByteArray beginValue, ByteArray endValue) { - Set<ByteArray> result = Sets.newHashSetWithExpectedSize(equalValues.size()); - for (ByteArray v : equalValues) { - if (rangeStartEndComparator.comparator.compare(beginValue, v) <= 0 && rangeStartEndComparator.comparator.compare(v, endValue) <= 0) { - result.add(v); - } - } - return equalValues; - } - - public String toString() { - if (valueSet == null) { - return column.getName() + " between " + begin + " and " + end; - } else { - return column.getName() + " in " + valueSet; - } - } - } - - public static abstract class ComparatorEx<T> implements Comparator<T> { - - public T min(Collection<T> v) { - if (v.size() <= 0) { - return null; - } - - Iterator<T> iterator = v.iterator(); - T min = iterator.next(); - while (iterator.hasNext()) { - min = min(min, iterator.next()); - } - return min; - } - - public T max(Collection<T> v) { - if (v.size() <= 0) { - return null; - } - - Iterator<T> iterator = v.iterator(); - T max = iterator.next(); - while (iterator.hasNext()) { - max = max(max, iterator.next()); - } - return max; - } - - public T min(T a, T b) { - return compare(a, b) <= 0 ? a : b; - } - - public T max(T a, T b) { - return compare(a, b) >= 0 ? a : b; - } - - public boolean between(T v, T start, T end) { - return compare(start, v) <= 0 && compare(v, end) <= 0; - } - } - - public static RecordComparator getRangeStartComparator(final IGTComparator comp) { - return new RecordComparator(new ComparatorEx<ByteArray>() { - @Override - public int compare(ByteArray a, ByteArray b) { - if (a.array() == null) { - if (b.array() == null) { - return 0; - } else { - return -1; - } - } else if (b.array() == null) { - return 1; - } else { - return comp.compare(a, b); - } - } - }); - } - - public static RecordComparator getRangeEndComparator(final IGTComparator comp) { - return new RecordComparator(new ComparatorEx<ByteArray>() { - @Override - public int compare(ByteArray a, ByteArray b) { - if (a.array() == null) { - if (b.array() == null) { - return 0; - } else { - return 1; - } - } else if (b.array() == null) { - return -1; - } else { - return comp.compare(a, b); - } - } - }); - } - - public static RecordComparator getRangeStartEndComparator(final IGTComparator comp) { - return new AsymmetricRecordComparator(new ComparatorEx<ByteArray>() { - @Override - public int compare(ByteArray a, ByteArray b) { - if (a.array() == null || b.array() == null) { - return -1; - } else { - return comp.compare(a, b); - } - } - }); - } - - private static class RecordComparator extends ComparatorEx<GTRecord> { - final ComparatorEx<ByteArray> comparator; - - RecordComparator(ComparatorEx<ByteArray> byteComparator) { - this.comparator = byteComparator; - } - - @Override - public int compare(GTRecord a, GTRecord b) { - assert a.getInfo() == b.getInfo(); - - int comp; - ImmutableBitSet allColumns = a.getInfo().getAllColumns(); - for (int i = 0; i < allColumns.trueBitCount(); i++) { - int c = allColumns.trueBitAt(i); - comp = comparator.compare(a.get(c), b.get(c)); - if (comp != 0) - return comp; - } - return 0; // equals - } - } - - /** - * asymmetric means compare(a,b) > 0 does not cause compare(b,a) < 0 - * so min max functions will not be supported - */ - private static class AsymmetricRecordComparator extends RecordComparator { - - AsymmetricRecordComparator(ComparatorEx<ByteArray> byteComparator) { - super(byteComparator); - } - - public GTRecord min(Collection<GTRecord> v) { - throw new UnsupportedOperationException(); - } - - public GTRecord max(Collection<GTRecord> v) { - throw new UnsupportedOperationException(); - } - - public GTRecord min(GTRecord a, GTRecord b) { - throw new UnsupportedOperationException(); - } - - public GTRecord max(GTRecord a, GTRecord b) { - throw new UnsupportedOperationException(); - } - - public boolean between(GTRecord v, GTRecord start, GTRecord end) { - throw new UnsupportedOperationException(); - } - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java index 0904686..2e5dd12 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java @@ -20,9 +20,12 @@ package org.apache.kylin.cube.gridtable; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.cube.cuboid.Cuboid; @@ -120,16 +123,8 @@ public class CuboidToGridTableMapping { return nDimensions + nMetrics; } - public int getDimensionCount() { - return nDimensions; - } - - public int getMetricsCount() { - return nMetrics; - } - public DataType[] getDataTypes() { - return (DataType[]) gtDataTypes.toArray(new DataType[gtDataTypes.size()]); + return gtDataTypes.toArray(new DataType[gtDataTypes.size()]); } public ImmutableBitSet getPrimaryKey() { @@ -137,7 +132,7 @@ public class CuboidToGridTableMapping { } public ImmutableBitSet[] getColumnBlocks() { - return (ImmutableBitSet[]) gtColBlocks.toArray(new ImmutableBitSet[gtColBlocks.size()]); + return gtColBlocks.toArray(new ImmutableBitSet[gtColBlocks.size()]); } public int getIndexOf(TblColRef dimension) { @@ -147,7 +142,7 @@ public class CuboidToGridTableMapping { public int getIndexOf(FunctionDesc metric) { Integer r = metrics2gt.get(metric); - return r == null ? -1 : r.intValue(); + return r == null ? -1 : r; } public List<TblColRef> getCuboidDimensionsInGTOrder() { @@ -185,4 +180,47 @@ public class CuboidToGridTableMapping { return result.isEmpty() ? Collections.<Integer, Integer> emptyMap() : result; } + public ImmutableBitSet makeGridTableColumns(Set<TblColRef> dimensions) { + BitSet result = new BitSet(); + for (TblColRef dim : dimensions) { + int idx = this.getIndexOf(dim); + if (idx >= 0) + result.set(idx); + } + return new ImmutableBitSet(result); + } + + public ImmutableBitSet makeGridTableColumns(Collection<FunctionDesc> metrics) { + BitSet result = new BitSet(); + for (FunctionDesc metric : metrics) { + int idx = this.getIndexOf(metric); + if (idx < 0) + throw new IllegalStateException(metric + " not found in " + this); + result.set(idx); + } + return new ImmutableBitSet(result); + } + + public String[] makeAggrFuncs(Collection<FunctionDesc> metrics) { + + //metrics are represented in ImmutableBitSet, which loses order information + //sort the aggrFuns to align with metrics natural order + List<FunctionDesc> metricList = Lists.newArrayList(metrics); + Collections.sort(metricList, new Comparator<FunctionDesc>() { + @Override + public int compare(FunctionDesc o1, FunctionDesc o2) { + int a = CuboidToGridTableMapping.this.getIndexOf(o1); + int b = CuboidToGridTableMapping.this.getIndexOf(o2); + return a - b; + } + }); + + String[] result = new String[metricList.size()]; + int i = 0; + for (FunctionDesc metric : metricList) { + result[i++] = metric.getExpression(); + } + return result; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java new file mode 100644 index 0000000..5360822 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.gridtable; + +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.ImmutableBitSet; +import org.apache.kylin.gridtable.GTRecord; + +public class RecordComparator extends ComparatorEx<GTRecord> { + final public ComparatorEx<ByteArray> comparator; + + RecordComparator(ComparatorEx<ByteArray> byteComparator) { + this.comparator = byteComparator; + } + + @Override + public int compare(GTRecord a, GTRecord b) { + assert a.getInfo() == b.getInfo(); + + int comp; + ImmutableBitSet allColumns = a.getInfo().getAllColumns(); + for (int i = 0; i < allColumns.trueBitCount(); i++) { + int c = allColumns.trueBitAt(i); + comp = comparator.compare(a.get(c), b.get(c)); + if (comp != 0) + return comp; + } + return 0; // equals + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java new file mode 100644 index 0000000..43ea30b --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.gridtable; + +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.gridtable.IGTComparator; + +public class RecordComparators { + + public static RecordComparator getRangeStartComparator(final IGTComparator comp) { + return new RecordComparator(new ComparatorEx<ByteArray>() { + @Override + public int compare(ByteArray a, ByteArray b) { + if (a.array() == null) { + if (b.array() == null) { + return 0; + } else { + return -1; + } + } else if (b.array() == null) { + return 1; + } else { + return comp.compare(a, b); + } + } + }); + } + + public static RecordComparator getRangeEndComparator(final IGTComparator comp) { + return new RecordComparator(new ComparatorEx<ByteArray>() { + @Override + public int compare(ByteArray a, ByteArray b) { + if (a.array() == null) { + if (b.array() == null) { + return 0; + } else { + return 1; + } + } else if (b.array() == null) { + return -1; + } else { + return comp.compare(a, b); + } + } + }); + } + + public static RecordComparator getRangeStartEndComparator(final IGTComparator comp) { + return new AsymmetricRecordComparator(new ComparatorEx<ByteArray>() { + @Override + public int compare(ByteArray a, ByteArray b) { + if (a.array() == null || b.array() == null) { + return -1; + } else { + return comp.compare(a, b); + } + } + }); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java new file mode 100644 index 0000000..d938f2b --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.gridtable; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.ImmutableBitSet; +import org.apache.kylin.common.util.Pair; +import org.apache.kylin.gridtable.GTInfo; +import org.apache.kylin.gridtable.GTScanRequest; +import org.apache.kylin.metadata.filter.CompareTupleFilter; +import org.apache.kylin.metadata.filter.ConstantTupleFilter; +import org.apache.kylin.metadata.filter.LogicalTupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter; +import org.apache.kylin.metadata.model.TblColRef; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +public abstract class ScanRangePlannerBase { + + //GT + protected TupleFilter gtFilter; + protected GTInfo gtInfo; + protected Pair<ByteArray, ByteArray> gtStartAndEnd; + protected TblColRef gtPartitionCol; + protected ImmutableBitSet gtDimensions; + protected ImmutableBitSet gtAggrGroups; + protected ImmutableBitSet gtAggrMetrics; + protected String[] gtAggrFuncs; + protected boolean isPartitionColUsingDatetimeEncoding = true; + + protected RecordComparator rangeStartComparator; + protected RecordComparator rangeEndComparator; + protected RecordComparator rangeStartEndComparator; + + public abstract GTScanRequest planScanRequest(); + + protected TupleFilter flattenToOrAndFilter(TupleFilter filter) { + if (filter == null) + return null; + + TupleFilter flatFilter = filter.flatFilter(); + + // normalize to OR-AND filter + if (flatFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) { + LogicalTupleFilter f = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR); + f.addChild(flatFilter); + flatFilter = f; + } + + if (flatFilter.getOperator() != TupleFilter.FilterOperatorEnum.OR) + throw new IllegalStateException(); + + return flatFilter; + } + + protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) { + List<Collection<ColumnRange>> result = Lists.newArrayList(); + + if (flatFilter == null) { + result.add(Collections.<ColumnRange> emptyList()); + return result; + } + + for (TupleFilter andFilter : flatFilter.getChildren()) { + if (andFilter.getOperator() != TupleFilter.FilterOperatorEnum.AND) + throw new IllegalStateException("Filter should be AND instead of " + andFilter); + + Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren()); + if (andRanges != null) { + result.add(andRanges); + } + } + + return preEvaluateConstantConditions(result); + } + + private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) { + Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>(); + for (TupleFilter filter : andFilters) { + if ((filter instanceof CompareTupleFilter) == false) { + if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) { + return null; + } else { + continue; + } + } + + CompareTupleFilter comp = (CompareTupleFilter) filter; + if (comp.getColumn() == null) { + continue; + } + + @SuppressWarnings("unchecked") + ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(), comp.getOperator()); + ColumnRange existing = rangeMap.get(newRange.column); + if (existing == null) { + rangeMap.put(newRange.column, newRange); + } else { + existing.andMerge(newRange); + } + } + return rangeMap.values(); + } + + private List<Collection<ColumnRange>> preEvaluateConstantConditions(List<Collection<ColumnRange>> orAndRanges) { + boolean globalAlwaysTrue = false; + Iterator<Collection<ColumnRange>> iterator = orAndRanges.iterator(); + while (iterator.hasNext()) { + Collection<ColumnRange> andRanges = iterator.next(); + Iterator<ColumnRange> iterator2 = andRanges.iterator(); + boolean hasAlwaysFalse = false; + while (iterator2.hasNext()) { + ColumnRange range = iterator2.next(); + if (range.satisfyAll()) + iterator2.remove(); + else if (range.satisfyNone()) + hasAlwaysFalse = true; + } + if (hasAlwaysFalse) { + iterator.remove(); + } else if (andRanges.isEmpty()) { + globalAlwaysTrue = true; + break; + } + } + // return empty OR list means global false + // return an empty AND collection inside OR list means global true + if (globalAlwaysTrue) { + orAndRanges.clear(); + orAndRanges.add(Collections.<ColumnRange> emptyList()); + } + return orAndRanges; + } + + public class ColumnRange { + public TblColRef column; + public ByteArray begin = ByteArray.EMPTY; + public ByteArray end = ByteArray.EMPTY; + public Set<ByteArray> valueSet; + public boolean isBoundryInclusive; + + public ColumnRange(TblColRef column, Set<ByteArray> values, TupleFilter.FilterOperatorEnum op) { + this.column = column; + + //TODO: the treatment is un-precise + if (op == TupleFilter.FilterOperatorEnum.EQ || op == TupleFilter.FilterOperatorEnum.IN || op == TupleFilter.FilterOperatorEnum.LTE || op == TupleFilter.FilterOperatorEnum.GTE) { + isBoundryInclusive = true; + } + + switch (op) { + case EQ: + case IN: + valueSet = new HashSet<ByteArray>(values); + refreshBeginEndFromEquals(); + break; + case LT: + case LTE: + end = rangeEndComparator.comparator.max(values); + break; + case GT: + case GTE: + begin = rangeStartComparator.comparator.min(values); + break; + case NEQ: + case NOTIN: + case ISNULL: + case ISNOTNULL: + // let Optiq filter it! + break; + default: + throw new UnsupportedOperationException(op.name()); + } + } + + void copy(TblColRef column, ByteArray beginValue, ByteArray endValue, Set<ByteArray> equalValues) { + this.column = column; + this.begin = beginValue; + this.end = endValue; + this.valueSet = equalValues; + } + + private void refreshBeginEndFromEquals() { + if (valueSet.isEmpty()) { + begin = ByteArray.EMPTY; + end = ByteArray.EMPTY; + } else { + begin = rangeStartComparator.comparator.min(valueSet); + end = rangeEndComparator.comparator.max(valueSet); + } + } + + public boolean satisfyAll() { + return begin.array() == null && end.array() == null; // the NEQ case + } + + public boolean satisfyNone() { + if (valueSet != null) { + return valueSet.isEmpty(); + } else if (begin.array() != null && end.array() != null) { + return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0; + } else { + return false; + } + } + + public void andMerge(ColumnRange another) { + assert this.column.equals(another.column); + + if (another.satisfyAll()) { + return; + } + + if (this.satisfyAll()) { + copy(another.column, another.begin, another.end, another.valueSet); + return; + } + + if (this.valueSet != null && another.valueSet != null) { + this.valueSet.retainAll(another.valueSet); + refreshBeginEndFromEquals(); + return; + } + + if (this.valueSet != null) { + this.valueSet = filter(this.valueSet, another.begin, another.end); + refreshBeginEndFromEquals(); + return; + } + + if (another.valueSet != null) { + this.valueSet = filter(another.valueSet, this.begin, this.end); + refreshBeginEndFromEquals(); + return; + } + + this.begin = rangeStartComparator.comparator.max(this.begin, another.begin); + this.end = rangeEndComparator.comparator.min(this.end, another.end); + this.isBoundryInclusive |= another.isBoundryInclusive; + } + + private Set<ByteArray> filter(Set<ByteArray> equalValues, ByteArray beginValue, ByteArray endValue) { + Set<ByteArray> result = Sets.newHashSetWithExpectedSize(equalValues.size()); + for (ByteArray v : equalValues) { + if (rangeStartEndComparator.comparator.compare(beginValue, v) <= 0 && rangeStartEndComparator.comparator.compare(v, endValue) <= 0) { + result.add(v); + } + } + return equalValues; + } + + public String toString() { + if (valueSet == null) { + return column.getName() + " between " + begin + " and " + end; + } else { + return column.getName() + " in " + valueSet; + } + } + + } + + protected String makeReadable(ByteArray byteArray) { + if (byteArray == null) { + return null; + } else { + return byteArray.toReadableText(); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java new file mode 100644 index 0000000..e31111d --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.gridtable; + +import java.nio.ByteBuffer; + +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.DateFormat; +import org.apache.kylin.common.util.Pair; +import org.apache.kylin.cube.ISegment; +import org.apache.kylin.dimension.AbstractDateDimEnc; +import org.apache.kylin.gridtable.GTInfo; +import org.apache.kylin.metadata.datatype.DataType; + +public class SegmentGTStartAndEnd { + private ISegment segment; + private GTInfo info; + + public SegmentGTStartAndEnd(ISegment segment, GTInfo info) { + this.segment = segment; + this.info = info; + } + + public boolean isUsingDatetimeEncoding(int index) { + return info.getCodeSystem().getDimEnc(index) instanceof AbstractDateDimEnc; + } + + public Pair<ByteArray, ByteArray> getSegmentStartAndEnd(int index) { + ByteArray start; + if (segment.getDateRangeStart() != Long.MIN_VALUE) { + start = encodeTime(segment.getDateRangeStart(), index, 1); + } else { + start = new ByteArray(); + } + + ByteArray end; + if (segment.getDateRangeEnd() != Long.MAX_VALUE) { + end = encodeTime(segment.getDateRangeEnd(), index, -1); + } else { + end = new ByteArray(); + } + return Pair.newPair(start, end); + + } + + private ByteArray encodeTime(long ts, int index, int roundingFlag) { + String value; + DataType partitionColType = info.getColumnType(index); + if (partitionColType.isDate()) { + value = DateFormat.formatToDateStr(ts); + } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) { + value = DateFormat.formatToTimeWithoutMilliStr(ts); + } else if (partitionColType.isStringFamily()) { + String partitionDateFormat = segment.getModel().getPartitionDesc().getPartitionDateFormat(); + if (StringUtils.isEmpty(partitionDateFormat)) + partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN; + value = DateFormat.formatToDateStr(ts, partitionDateFormat); + } else { + throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); + } + + ByteBuffer buffer = ByteBuffer.allocate(info.getMaxColumnLength()); + info.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer); + + return ByteArray.copyOf(buffer.array(), 0, buffer.position()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java index 21da4ea..12a3593 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java @@ -31,8 +31,11 @@ import org.apache.kylin.cube.gridtable.CubeCodeSystem; import org.apache.kylin.cube.gridtable.TrimmedCubeCodeSystem; import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GTInfo { + private static final Logger logger = LoggerFactory.getLogger(GTInfo.class); public static Builder builder() { return new Builder(); @@ -151,7 +154,7 @@ public class GTInfo { if (!expected.equals(ref)) throw new IllegalArgumentException(); } - + void validate() { if (codeSystem == null) throw new IllegalStateException(); @@ -271,7 +274,7 @@ public class GTInfo { public IGTCodeSystem getCodeSystem() { return codeSystem; } - + public int getMaxLength() { int ret = 0; for (int i = 0; i < colAll.trueBitCount(); i++) { @@ -291,7 +294,7 @@ public class GTInfo { BytesUtil.writeAsciiString(GTSampleCodeSystem.class.getCanonicalName(), out); GTSampleCodeSystem.serializer.serialize((GTSampleCodeSystem) value.codeSystem, out); } else { - throw new IllegalArgumentException("Can't recognize code system " + value.codeSystem.getClass()); + BytesUtil.writeAsciiString(value.codeSystem.getClass().getCanonicalName(), out); } BytesUtil.writeUTFString(value.tableName, out); @@ -317,7 +320,11 @@ public class GTInfo { } else if (GTSampleCodeSystem.class.getCanonicalName().equals(codeSystemType)) { codeSystem = GTSampleCodeSystem.serializer.deserialize(in); } else { - throw new IllegalArgumentException("Can't recognize code system " + codeSystemType); + try { + codeSystem = (IGTCodeSystem) Class.forName(codeSystemType).newInstance(); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + throw new RuntimeException(e); + } } String newTableName = BytesUtil.readUTFString(in); @@ -349,5 +356,4 @@ public class GTInfo { } }; - } http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index e2bac3d..4cfba1b 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -54,8 +54,8 @@ public class GTScanRequest { // optional aggregation private ImmutableBitSet aggrGroupBy; private ImmutableBitSet aggrMetrics; - private String[] aggrMetricsFuncs; - + private String[] aggrMetricsFuncs;// + // hint to storage behavior private boolean allowStorageAggregation; private double aggCacheMemThreshold; @@ -204,7 +204,7 @@ public class GTScanRequest { //TODO BUG? select sum() from fact, no aggr by public boolean hasAggregation() { - return aggrGroupBy != null && aggrMetrics != null && aggrMetricsFuncs != null; + return !aggrGroupBy.isEmpty() || !aggrMetrics.isEmpty(); } public GTInfo getInfo() { @@ -284,16 +284,6 @@ public class GTScanRequest { this.storagePushDownLimit = limit; } - public List<Integer> getRequiredMeasures() { - List<Integer> measures = Lists.newArrayList(); - int numDim = info.getPrimaryKey().trueBitCount(); - for (int i = 0; i < aggrMetrics.trueBitCount(); i++) { - int index = aggrMetrics.trueBitAt(i); - measures.add(index - numDim); - } - return measures; - } - @Override public String toString() { return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]"; http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java index 49ec759..c4390cd 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java @@ -18,6 +18,7 @@ package org.apache.kylin.gridtable; +import java.util.BitSet; import java.util.List; import org.apache.kylin.common.util.ImmutableBitSet; @@ -92,6 +93,18 @@ public class GTScanRequestBuilder { } public GTScanRequest createGTScanRequest() { + if (aggrGroupBy == null) { + aggrGroupBy = new ImmutableBitSet(new BitSet()); + } + + if (aggrMetrics == null) { + aggrMetrics = new ImmutableBitSet(new BitSet()); + } + + if (aggrMetricsFuncs == null) { + aggrMetricsFuncs = new String[0]; + } + return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, storagePushDownLimit); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java index 5be91be..bb7503a 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java @@ -24,7 +24,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Iterator; -import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.cube.ISegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,9 +34,9 @@ public class ScannerWorker { private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class); private IGTScanner internal = null; - public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) { + public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) { if (scanRequest == null) { - logger.info("Segment {} will be skipped", cubeSeg); + logger.info("Segment {} will be skipped", segment); internal = new EmptyGTScanner(0); return; } @@ -44,7 +44,7 @@ public class ScannerWorker { final GTInfo info = scanRequest.getInfo(); try { - IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(CubeSegment.class, Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // default behavior + IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(ISegment.class, Cuboid.class, GTInfo.class).newInstance(segment, cuboid, info); // default behavior internal = rpc.getGTScanner(scanRequest); } catch (IOException | InstantiationException | InvocationTargetException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException e) { throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java index 2bbf6b7..7b6d3fa 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java @@ -132,7 +132,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { LogicalTupleFilter filter = and(timeComp4, ageComp1); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); - assertEquals(1, r.size()); + assertEquals(0, r.size()); } { LogicalTupleFilter filter = and(timeComp5, ageComp1); http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java index da2cf98..1f883bd 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java @@ -32,6 +32,7 @@ import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.RealizationRegistry; +import org.apache.kylin.metadata.realization.RealizationType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -206,6 +207,13 @@ class ProjectL2Cache { } else { logger.warn("Realization '" + entry + "' defined under project '" + project + "' is not found"); } + + //check if there's raw table parasite + //TODO: ugly impl here + IRealization parasite = registry.getRealization(RealizationType.INVERTED_INDEX, entry.getRealization()); + if (parasite != null) { + projectCache.realizations.add(parasite); + } } for (IRealization realization : projectCache.realizations) { http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java index aa90fc5..1eee1e7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java @@ -44,7 +44,6 @@ public class SQLDigest { public Collection<FunctionDesc> aggregations; public Collection<MeasureDesc> sortMeasures; public Collection<OrderEnum> sortOrders; - private boolean isRawQuery = false; public SQLDigest(String factTable, TupleFilter filter, Collection<JoinDesc> joinDescs, Collection<TblColRef> allColumns, // Collection<TblColRef> groupbyColumns, Collection<TblColRef> filterColumns, Collection<TblColRef> aggregatedColumns, Collection<FunctionDesc> aggregateFunnc, Collection<MeasureDesc> sortMeasures, Collection<OrderEnum> sortOrders) { @@ -58,11 +57,12 @@ public class SQLDigest { this.aggregations = aggregateFunnc; this.sortMeasures = sortMeasures; this.sortOrders = sortOrders; - this.isRawQuery = this.groupbyColumns.isEmpty() && this.metricColumns.isEmpty(); } public boolean isRawQuery() { - return isRawQuery; + return this.groupbyColumns.isEmpty() && // select a group by a -> not raw + this.aggregations.isEmpty(); // has aggr -> not raw + //the reason to choose aggregations rather than metricColumns is because the former is set earlier at implOLAP } @Override
