minor, code refactor in GTCubeStorageQueryBase
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4cce762 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4cce762 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4cce762 Branch: refs/heads/KYLIN-2506 Commit: d4cce762addda1c819939e767220864dad551503 Parents: f6ee76c Author: lidongsjtu <lid...@apache.org> Authored: Thu Mar 30 17:25:22 2017 +0800 Committer: hongbin ma <m...@kyligence.io> Committed: Fri Mar 31 18:34:37 2017 +0800 ---------------------------------------------------------------------- .../gtrecord/GTCubeStorageQueryBase.java | 68 +++++++------- .../gtrecord/GTCubeStorageQueryRequest.java | 94 ++++++++++++++++++++ 2 files changed, 130 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d4cce762/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index ecf1ad3..1b4964f 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -72,6 +72,30 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { @Override public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { + GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo); + + List<CubeSegmentScanner> scanners = Lists.newArrayList(); + for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) { + CubeSegmentScanner scanner; + + if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) { + logger.info("Skip cube segment {} because its input record is 0", cubeSeg); + continue; + } + + scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), request.getMetrics(), request.getFilter(), request.getContext()); + scanners.add(scanner); + } + + if (scanners.isEmpty()) + return ITupleIterator.EMPTY_TUPLE_ITERATOR; + + return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(), request.getMetrics(), returnTupleInfo, request.getContext()); + } + + protected abstract String getGTStorage(); + + protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { context.setStorageQuery(this); //deal with participant columns in subquery join @@ -106,7 +130,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { dimensionsD.addAll(groupsD); dimensionsD.addAll(otherDimsD); Cuboid cuboid = Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics); - logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, otherDimsD={}", cubeInstance.getName(), cuboid.getId(), groupsD, otherDimsD); context.setCuboid(cuboid); // set whether to aggr at storage @@ -115,35 +138,21 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { // replace derived columns in filter with host columns; columns on loosened condition must be added to group by Set<TblColRef> loosenedColumnD = Sets.newHashSet(); + Set<TblColRef> filterColumnD = Sets.newHashSet(); TupleFilter filterD = translateDerived(filter, loosenedColumnD); groupsD.addAll(loosenedColumnD); + TupleFilter.collectColumns(filter, filterColumnD); // set limit push down enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context); // set query deadline context.setDeadline(cubeInstance); - List<CubeSegmentScanner> scanners = Lists.newArrayList(); - for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) { - CubeSegmentScanner scanner; - - if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) { - logger.info("Skip cube segment {} because its input record is 0", cubeSeg); - continue; - } - - scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context); - scanners.add(scanner); - } + logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation()); - if (scanners.isEmpty()) - return ITupleIterator.EMPTY_TUPLE_ITERATOR; - - return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context); + return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, context); } - protected abstract String getGTStorage(); - protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo tupleInfo) { return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo); } @@ -235,19 +244,14 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD) { - - logger.info("GroupD :" + groupD); - logger.info("SingleValueD :" + singleValueD); - logger.info("Cuboid columns :" + cuboid.getColumns()); - HashSet<TblColRef> temp = Sets.newHashSet(); temp.addAll(groupD); temp.addAll(singleValueD); if (cuboid.getColumns().size() == temp.size()) { - logger.info("Does not need storage aggregation"); + logger.debug("Does not need storage aggregation"); return false; } else { - logger.info("Need storage aggregation"); + logger.debug("Need storage aggregation"); return true; } } @@ -326,30 +330,30 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { if (!TupleFilter.isEvaluableRecursively(filter)) { possible = false; - logger.info("Storage limit push down is impossible because the filter isn't evaluable"); + logger.debug("Storage limit push down is impossible because the filter isn't evaluable"); } if (!loosenedColumnD.isEmpty()) { // KYLIN-2173 possible = false; - logger.info("Storage limit push down is impossible because filter is loosened: " + loosenedColumnD); + logger.debug("Storage limit push down is impossible because filter is loosened: " + loosenedColumnD); } if (context.hasSort()) { possible = false; - logger.info("Storage limit push down is impossible because the query has order by"); + logger.debug("Storage limit push down is impossible because the query has order by"); } // derived aggregation is bad, unless expanded columns are already in group by if (!groups.containsAll(derivedPostAggregation)) { possible = false; - logger.info("Storage limit push down is impossible because derived column require post aggregation: " + derivedPostAggregation); + logger.debug("Storage limit push down is impossible because derived column require post aggregation: " + derivedPostAggregation); } //if groupsD is clustered at "head" of the rowkey, then limit push down is possible int size = groupsD.size(); if (!groupsD.containsAll(cuboid.getColumns().subList(0, size))) { possible = false; - logger.info("Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD // + logger.debug("Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD // + " with cuboid columns: " + cuboid.getColumns()); } @@ -357,7 +361,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { for (FunctionDesc functionDesc : functionDescs) { if (functionDesc.isDimensionAsMetric()) { possible = false; - logger.info("Storage limit push down is impossible because {} isDimensionAsMetric ", functionDesc); + logger.debug("Storage limit push down is impossible because {} isDimensionAsMetric ", functionDesc); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/d4cce762/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java new file mode 100644 index 0000000..c4d81d2 --- /dev/null +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.storage.gtrecord; + +import java.io.Serializable; +import java.util.Set; + +import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.metadata.filter.TupleFilter; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.storage.StorageContext; + +public class GTCubeStorageQueryRequest implements Serializable { + private Cuboid cuboid; + private Set<TblColRef> dimensions; + private Set<TblColRef> groups; + private Set<FunctionDesc> metrics; + private TupleFilter filter; + private StorageContext context; + + public GTCubeStorageQueryRequest(Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics, TupleFilter filter, StorageContext context) { + this.cuboid = cuboid; + this.dimensions = dimensions; + this.groups = groups; + this.metrics = metrics; + this.filter = filter; + this.context = context; + } + + public Cuboid getCuboid() { + return cuboid; + } + + public void setCuboid(Cuboid cuboid) { + this.cuboid = cuboid; + } + + public Set<TblColRef> getDimensions() { + return dimensions; + } + + public void setDimensions(Set<TblColRef> dimensions) { + this.dimensions = dimensions; + } + + public Set<TblColRef> getGroups() { + return groups; + } + + public void setGroups(Set<TblColRef> groups) { + this.groups = groups; + } + + public Set<FunctionDesc> getMetrics() { + return metrics; + } + + public void setMetrics(Set<FunctionDesc> metrics) { + this.metrics = metrics; + } + + public TupleFilter getFilter() { + return filter; + } + + public void setFilter(TupleFilter filter) { + this.filter = filter; + } + + public StorageContext getContext() { + return context; + } + + public void setContext(StorageContext context) { + this.context = context; + } +}
