KYLIN-2290 minor improvements on limit KYLIN-2290 fix compile
KYLIN-2290 bug fix Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f9bd4a9 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f9bd4a9 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f9bd4a9 Branch: refs/heads/master-cdh5.7 Commit: 6f9bd4a9998cff24c97bce0e6d86282f48fef388 Parents: 699a88e Author: Hongbin Ma <mahong...@apache.org> Authored: Fri Dec 16 16:28:18 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Mon Dec 19 14:16:00 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 1 + .../kylin/cube/CubeCapabilityChecker.java | 5 ++ .../kylin/gridtable/GTScanRequestBuilder.java | 2 +- .../kylin/metadata/realization/SQLDigest.java | 4 +- .../apache/kylin/storage/StorageContext.java | 74 +++++++------------- .../storage/gtrecord/CubeScanRangePlanner.java | 9 +-- .../gtrecord/GTCubeStorageQueryBase.java | 1 - .../gtrecord/SequentialCubeTupleIterator.java | 6 +- .../apache/kylin/query/ITKylinQueryTest.java | 1 - .../org/apache/kylin/query/KylinTestBase.java | 7 +- .../kylin/storage/hbase/ITStorageTest.java | 2 +- .../kylin/query/relnode/OLAPAggregateRel.java | 4 ++ .../apache/kylin/query/relnode/OLAPContext.java | 6 +- .../kylin/query/relnode/OLAPFilterRel.java | 2 +- .../kylin/query/relnode/OLAPLimitRel.java | 6 +- .../coprocessor/endpoint/CubeVisitService.java | 7 +- 16 files changed, 60 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 610c2af..5153562 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -778,6 +778,7 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true")); } + @Deprecated//Limit is good even it's large. This config is meaning less since we already have scan threshold public int getStoragePushDownLimitMax() { return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index 38faed9..c45144b 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -120,6 +120,11 @@ public class CubeCapabilityChecker { return result; } + if (digest.limitPrecedesAggr) { + logger.info("Exclude cube " + cube.getName() + " because there's limit preceding aggregation"); + return result; + } + if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) { result.influences.add(new CapabilityInfluence() { @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java index f542de1..bcec1f4 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java @@ -36,7 +36,7 @@ public class GTScanRequestBuilder { private boolean allowStorageAggregation = true; private double aggCacheMemThreshold = 0; private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage should terminate itself when $storageScanRowNumThreshold cuboid rows are scanned, and throw exception. - private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit working when $toragePushDownLimit aggregated rows are produced. + private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit scanning safely when $toragePushDownLimit aggregated rows are produced. private long startTime = -1; private long timeout = -1; private String storageBehavior = null; http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java index 83fc05c..36f303b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java @@ -59,10 +59,11 @@ public class SQLDigest { public List<TblColRef> sortColumns; public List<OrderEnum> sortOrders; public boolean isRawQuery; + public boolean limitPrecedesAggr; public SQLDigest(String factTable, TupleFilter filter, List<JoinDesc> joinDescs, Set<TblColRef> allColumns, // List<TblColRef> groupbyColumns, Set<TblColRef> subqueryJoinParticipants, Set<TblColRef> filterColumns, Set<TblColRef> metricColumns, // - List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders) { + List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders, boolean limitPrecedesAggr) { this.factTable = factTable; this.filter = filter; this.joinDescs = joinDescs; @@ -76,6 +77,7 @@ public class SQLDigest { this.sortColumns = sortColumns; this.sortOrders = sortOrders; this.isRawQuery = isRawQuery(); + this.limitPrecedesAggr = limitPrecedesAggr; } private boolean isRawQuery() { http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java index bc43a87..9ef59fd 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java @@ -34,43 +34,29 @@ import com.google.common.collect.Range; public class StorageContext { private static final Logger logger = LoggerFactory.getLogger(StorageContext.class); - public static final int DEFAULT_THRESHOLD = 1000000; - private String connUrl; private int threshold; - private int limit; - private int offset; - private int finalPushDownLimit; - private boolean hasSort; - private boolean acceptPartialResult; - - private boolean exactAggregation; - private boolean needStorageAggregation; - private boolean enableLimit; - private boolean enableCoprocessor; - - private AtomicLong totalScanCount; + private int limit = Integer.MAX_VALUE; + private int offset = 0; + private int finalPushDownLimit = Integer.MAX_VALUE; + private boolean hasSort = false; + private boolean acceptPartialResult = false; + + private boolean exactAggregation = false; + private boolean needStorageAggregation = false; + private boolean limitEnabled = false; + private boolean enableCoprocessor = false; + + private AtomicLong totalScanCount = new AtomicLong(); private Cuboid cuboid; - private boolean partialResultReturned; - - private Range<Long> reusedPeriod; + private boolean partialResultReturned = false; public StorageContext() { - this.threshold = DEFAULT_THRESHOLD; - this.limit = DEFAULT_THRESHOLD; - this.totalScanCount = new AtomicLong(); - this.cuboid = null; - this.hasSort = false; - - this.exactAggregation = false; - this.enableLimit = false; - this.enableCoprocessor = false; - - this.acceptPartialResult = false; - this.partialResultReturned = false; - this.finalPushDownLimit = Integer.MAX_VALUE; + this.threshold = KylinConfig.getInstanceFromEnv().getScanThreshold(); } + private Range<Long> reusedPeriod; + public String getConnUrl() { return connUrl; } @@ -92,11 +78,10 @@ public class StorageContext { } public void setLimit(int l) { - if (l > limit) { - //cases like : select price from (select * from kylin_sales limit 10) limit 5000 - logger.info("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit); + if (limit != Integer.MAX_VALUE) { + logger.warn("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit); } else { - this.limit = l; + limit = l; } } @@ -109,15 +94,11 @@ public class StorageContext { } public void enableLimit() { - this.enableLimit = true; + this.limitEnabled = true; } public boolean isLimitEnabled() { - return this.enableLimit; - } - - private int getStoragePushDownLimit() { - return this.isLimitEnabled() ? this.getOffset() + this.getLimit() : Integer.MAX_VALUE; + return this.limitEnabled; } public int getFinalPushDownLimit() { @@ -126,19 +107,16 @@ public class StorageContext { public void setFinalPushDownLimit(IRealization realization) { - //decide the final limit push down - int tempPushDownLimit = this.getStoragePushDownLimit(); - if (tempPushDownLimit == Integer.MAX_VALUE) { + if (this.getLimit() == Integer.MAX_VALUE) { return; } - int pushDownLimitMax = KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax(); + int tempPushDownLimit = this.getOffset() + this.getLimit(); + if (!realization.supportsLimitPushDown()) { - logger.info("Not enabling limit push down because cube storage type not supported"); - } else if (tempPushDownLimit > pushDownLimitMax) { - logger.info("Not enabling limit push down because the limit(including offset) {} is larger than kylin.query.max-limit-pushdown {}", // - tempPushDownLimit, pushDownLimitMax); + logger.warn("Not enabling limit push down because cube storage type not supported"); } else { + this.limitEnabled = true; this.finalPushDownLimit = tempPushDownLimit; logger.info("Enable limit: " + tempPushDownLimit); } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java index 8d5a3d4..b05a629 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java @@ -148,15 +148,10 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { GTScanRequest scanRequest; List<GTScanRange> scanRanges = this.planScanRanges(); if (scanRanges != null && scanRanges.size() != 0) { - GTScanRequestBuilder builder = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).// + scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).// setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).// setAllowStorageAggregation(context.isNeedStorageAggregation()).setAggCacheMemThreshold(cubeSegment.getCubeInstance().getConfig().getQueryCoprocessorMemGB()).// - setStorageScanRowNumThreshold(context.getThreshold()); - - if (context.getFinalPushDownLimit() != Integer.MAX_VALUE) - builder.setStoragePushDownLimit(context.getFinalPushDownLimit()); - - scanRequest = builder.createGTScanRequest(); + setStoragePushDownLimit(context.getFinalPushDownLimit()).setStorageScanRowNumThreshold(context.getThreshold()).createGTScanRequest(); } else { scanRequest = null; } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 31573d0..7b0cbb9 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -383,7 +383,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } if (possible) { - context.enableLimit(); context.setFinalPushDownLimit(cubeInstance); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java index bef0e88..3a64de7 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java @@ -46,7 +46,6 @@ public class SequentialCubeTupleIterator implements ITupleIterator { protected List<CubeSegmentScanner> scanners; protected List<SegmentCubeTupleIterator> segmentCubeTupleIterators; protected Iterator<ITuple> tupleIterator; - protected final int storagePushDownLimit; protected StorageContext context; private int scanCount; @@ -62,8 +61,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator { segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context)); } - this.storagePushDownLimit = context.getFinalPushDownLimit(); - if (storagePushDownLimit == Integer.MAX_VALUE) { + if (!context.isLimitEnabled()) { //normal case tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator()); } else { @@ -75,7 +73,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator { return input; } }); - tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, storagePushDownLimit, segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator(); + tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 7f04bbb..c2aeabd 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -331,7 +331,6 @@ public class ITKylinQueryTest extends KylinTestBase { List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql"); for (File sqlFile : sqlFiles) { runSQL(sqlFile, false, false); - assertTrue(checkLimitEnabled()); assertTrue(checkFinalPushDownLimit()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index ddb996c..47b9e37 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -653,14 +653,9 @@ public class KylinTestBase { } - protected boolean checkLimitEnabled() { - OLAPContext context = getFirstOLAPContext(); - return (context.storageContext.isLimitEnabled()); - } - protected boolean checkFinalPushDownLimit() { OLAPContext context = getFirstOLAPContext(); - return (context.storageContext.getFinalPushDownLimit() != Integer.MAX_VALUE); + return context.storageContext.isLimitEnabled(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java index db9d133..aea8bef 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java @@ -148,7 +148,7 @@ public class ITStorageTest extends HBaseMetadataTestCase { int count = 0; ITupleIterator iterator = null; try { - SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>()); + SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false); iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations)); while (iterator.hasNext()) { ITuple tuple = iterator.next(); http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java index f5fa74d..c457545 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java @@ -156,6 +156,10 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { this.context.groupByColumns.addAll(this.groups); this.context.aggregations.addAll(this.aggregations); this.context.afterAggregate = true; + + if (this.context.afterLimit) { + this.context.limitPrecedesAggr = true; + } } else { for (AggregateCall aggCall : aggCalls) { // check if supported by kylin http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java index 8278fb0..dde98a6 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java @@ -112,7 +112,9 @@ public class OLAPContext { public Set<OLAPTableScan> allTableScans = new HashSet<>(); public TupleInfo returnTupleInfo = null; public boolean afterAggregate = false; - public boolean afterSkippedFilter = false; + public boolean afterHavingClauseFilter = false; + public boolean afterLimit = false; + public boolean limitPrecedesAggr = false; public boolean afterJoin = false; public boolean hasJoin = false; @@ -148,7 +150,7 @@ public class OLAPContext { public SQLDigest getSQLDigest() { if (sqlDigest == null) - sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders); + sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders, limitPrecedesAggr); return sqlDigest; } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java index 8023df4..151131e 100755 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -306,7 +306,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel { if (!context.afterAggregate) { translateFilter(context); } else { - context.afterSkippedFilter = true;//having clause is skipped + context.afterHavingClauseFilter = true;//having clause is skipped } } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java index f0af863..8179807 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java @@ -77,7 +77,9 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel { this.columnRowType = buildColumnRowType(); this.context = implementor.getContext(); - if (!context.afterSkippedFilter) { + // ignore limit after having clause + // ignore limit after another limit, e.g. select A, count(*) from (select A,B from fact group by A,B limit 100) limit 10 + if (!context.afterHavingClauseFilter && !context.afterLimit) { Number limitValue = (Number) (((RexLiteral) localFetch).getValue()); int limit = limitValue.intValue(); this.context.storageContext.setLimit(limit); @@ -87,6 +89,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel { int offset = offsetValue.intValue(); this.context.storageContext.setOffset(offset); } + + context.afterLimit = true; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java index da9c932..38efecc 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java @@ -254,14 +254,15 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement @Override public boolean hasNext() { + counter++; + if (counter > scanReq.getStorageScanRowNumThreshold()) { - throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter); + throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter + ", consider increasing kylin.query.memory-budget-bytes and kylin.query.scan-threshold"); } if (counter % (10 * GTScanRequest.terminateCheckInterval) == 1) { - logger.info("Scanned " + counter + " rows from HBase."); + logger.info("scanning " + counter + "th row from HBase."); } - counter++; return allCellLists.hasNext(); }
