KYLIN-2238 Add query server scan threshold
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b00ec26 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b00ec26 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b00ec26 Branch: refs/heads/master-cdh5.7 Commit: 4b00ec26de1e1cc9bf9b2e55a1b162d123114df8 Parents: 20c3294 Author: kangkaisen <kangkai...@live.com> Authored: Tue Nov 29 19:51:39 2016 +0800 Committer: kangkaisen <kangkai...@163.com> Committed: Sat Jan 14 19:00:09 2017 +0800 ---------------------------------------------------------------------- .../storage/gtrecord/SequentialCubeTupleIterator.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4b00ec26/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java index ee868c7..c621215 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java @@ -26,6 +26,7 @@ import java.util.Set; import javax.annotation.Nullable; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -33,6 +34,7 @@ import org.apache.kylin.metadata.tuple.ITuple; import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.metadata.tuple.TupleInfo; import org.apache.kylin.storage.StorageContext; +import org.apache.kylin.storage.exception.ScanOutOfLimitException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +47,8 @@ public class SequentialCubeTupleIterator implements ITupleIterator { private static final Logger logger = LoggerFactory.getLogger(SequentialCubeTupleIterator.class); + private final int SCAN_THRESHOLD = KylinConfig.getInstanceFromEnv().getScanThreshold(); + protected List<CubeSegmentScanner> scanners; protected List<SegmentCubeTupleIterator> segmentCubeTupleIterators; protected Iterator<ITuple> tupleIterator; @@ -78,7 +82,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator { tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, returnTupleInfo)).getIterator(); } } - + public Comparator<ITuple> getTupleDimensionComparator(Cuboid cuboid, TupleInfo returnTupleInfo) { // dimensionIndexOnTuple is for SQL with limit List<Integer> temp = Lists.newArrayList(); @@ -92,7 +96,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator { for (int i = 0; i < temp.size(); i++) { dimensionIndexOnTuple[i] = temp.get(i); } - + return new Comparator<ITuple>() { @Override public int compare(ITuple o1, ITuple o2) { @@ -137,7 +141,11 @@ public class SequentialCubeTupleIterator implements ITupleIterator { @Override public ITuple next() { - scanCount++; + // prevent the big query to make the Query Server OOM + if (scanCount++ > SCAN_THRESHOLD) { + throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " + SCAN_THRESHOLD); + } + if (++scanCountDelta >= 1000) flushScanCountDelta();
