Repository: kylin Updated Branches: refs/heads/yang21-hbase1.x 130c26df9 -> b28fe630b (forced update)
KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers in the rowkey Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61335d9c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61335d9c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61335d9c Branch: refs/heads/yang21-hbase1.x Commit: 61335d9ce8ff193bd22e881855d0c68e48896ca9 Parents: c23205d Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Nov 14 09:12:51 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Mon Nov 14 09:12:51 2016 +0800 ---------------------------------------------------------------------- .../gtrecord/GTCubeStorageQueryBase.java | 22 ++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/61335d9c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index b51af59..d648469 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -34,7 +34,9 @@ import org.apache.kylin.cube.RawQueryLastHacker; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; +import org.apache.kylin.cube.model.RowKeyColDesc; import org.apache.kylin.dict.lookup.LookupStringTable; +import org.apache.kylin.dimension.FixedLenDimEnc; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; @@ -120,7 +122,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { //set whether to aggr at storage context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, groupsD, singleValuesD)); // set limit push down - enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, context); + enableStorageLimitIfPossible(cubeInstance.getDescriptor(), cuboid, groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, context); context.setFinalPushDownLimit(cubeInstance); // set cautious threshold to prevent out of memory setThresholdIfNecessary(dimensionsD, metrics, context); @@ -412,9 +414,25 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } } - private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, Collection<FunctionDesc> functionDescs, StorageContext context) { + private void enableStorageLimitIfPossible(CubeDesc cubeDesc, Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, Collection<FunctionDesc> functionDescs, StorageContext context) { boolean possible = true; + for (TblColRef col : cuboid.getColumns()) { + RowKeyColDesc rowKeyDesc = cubeDesc.getRowkey().getColDesc(col); + if (rowKeyDesc != null) { + String encodingName = rowKeyDesc.getEncodingName(); + + //for numbers, if it's fixed_length encoding, order is no longer preserved + if (encodingName != null && encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && // + (col.getType().isNumberFamily() || col.getType().isIntegerFamily())) { + possible = false; + logger.info("Storage limit push down is impossible because integer/number is encoded as fixedlength"); + } + } else { + logger.warn("RowKeyColDesc for {} does not exist", col); + } + } + boolean goodFilter = filter == null || TupleFilter.isEvaluableRecursively(filter); if (!goodFilter) { possible = false;
