code refactor per KYLIN-2438
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4bdb62cb Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4bdb62cb Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4bdb62cb Branch: refs/heads/master-hbase0.98 Commit: 4bdb62cb3697e15d005a1a5383edf21f2a3ff567 Parents: 661f016 Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Feb 20 10:15:53 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Feb 21 20:58:31 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 6 +- .../org/apache/kylin/common/QueryContext.java | 2 +- .../cube/inmemcubing/ConcurrentDiskStore.java | 7 +- .../cube/inmemcubing/InMemCubeBuilder.java | 5 - .../kylin/cube/inmemcubing/MemDiskStore.java | 6 - .../apache/kylin/gridtable/EmptyGTScanner.java | 9 +- .../kylin/gridtable/GTAggregateScanner.java | 7 +- .../apache/kylin/gridtable/GTFilterScanner.java | 5 - .../apache/kylin/gridtable/GTScanRequest.java | 9 +- .../org/apache/kylin/gridtable/IGTScanner.java | 4 - .../benchmark/SortedGTRecordGenerator.java | 6 - .../gridtable/memstore/GTSimpleMemStore.java | 5 - .../gridtable/AggregationCacheSpillTest.java | 12 +- .../kylin/gridtable/SimpleGridTableTest.java | 9 +- .../storage/gtrecord/CubeSegmentScanner.java | 5 - .../kylin/storage/gtrecord/ScannerWorker.java | 15 +- .../gtrecord/StorageResponseGTScatter.java | 9 +- .../apache/kylin/query/ITFailfastQueryTest.java | 154 +++++++++++++++++++ .../apache/kylin/query/ITKylinQueryTest.java | 18 --- .../org/apache/kylin/query/KylinTestBase.java | 15 ++ .../apache/kylin/rest/service/QueryService.java | 12 +- .../hbase/cube/HBaseScannerBenchmark.java | 3 - .../storage/hbase/cube/SimpleHBaseStore.java | 5 - .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 22 ++- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 33 +--- .../hbase/cube/v2/HBaseReadonlyStore.java | 5 - .../coprocessor/endpoint/CubeVisitService.java | 2 +- .../endpoint/protobuf/CubeVisit.proto | 2 +- .../hbase/steps/SandboxMetastoreCLI.java | 1 - 29 files changed, 210 insertions(+), 183 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index f7d8452..13d967d 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -826,7 +826,7 @@ abstract public class KylinConfigBase implements Serializable { } public int getLargeQueryThreshold() { - return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf((int) (getScanThreshold() * 0.1)))); + return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf(1000000))); } public int getDerivedInThreshold() { @@ -865,6 +865,10 @@ abstract public class KylinConfigBase implements Serializable { return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-count", String.valueOf(10 * 1024))); } + public long getQueryScanBytesCacheThreshold() { + return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-bytes", String.valueOf(1024 * 1024))); + } + public boolean isQuerySecureEnabled() { return Boolean.parseBoolean(this.getOptional("kylin.query.security-enabled", "true")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-common/src/main/java/org/apache/kylin/common/QueryContext.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java index 67925b6..5457aa5 100644 --- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java +++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java @@ -49,7 +49,7 @@ public class QueryContext { } public String getQueryId() { - return queryId; + return queryId == null ? "" : queryId; } public void setQueryId(String queryId) { http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java index 5219ede..41d2dfb 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java @@ -265,11 +265,6 @@ public class ConcurrentDiskStore implements IGTStore, Closeable { return info; } - @Override - public long getScannedRowCount() { - return count; - } - } private class Writer implements IGTWriter { @@ -371,4 +366,4 @@ public class ConcurrentDiskStore implements IGTStore, Closeable { return "ConcurrentDiskStore@" + (info.getTableName() == null ? this.hashCode() : info.getTableName()); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index a74f0c0..e08844e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -564,10 +564,5 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return 0L; - } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java index 81403ab..a5471df 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java @@ -277,12 +277,6 @@ public class MemDiskStore implements IGTStore, Closeable { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return count; - } - } private class Writer implements IGTWriter { http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java index 01d31f0..8b6c995 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java @@ -22,10 +22,8 @@ import java.io.IOException; import java.util.Iterator; public class EmptyGTScanner implements IGTScanner { - private long reportScannedRowCount; - public EmptyGTScanner(long reportScannedRowCount) { - this.reportScannedRowCount = reportScannedRowCount; + public EmptyGTScanner() { } @Override @@ -34,11 +32,6 @@ public class EmptyGTScanner implements IGTScanner { } @Override - public long getScannedRowCount() { - return reportScannedRowCount; - } - - @Override public void close() throws IOException { } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java index 8b0efcc..7cdd4f5 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java @@ -129,11 +129,6 @@ public class GTAggregateScanner implements IGTScanner { } @Override - public long getScannedRowCount() { - return inputScanner.getScannedRowCount(); - } - - @Override public void close() throws IOException { inputScanner.close(); aggrCache.close(); @@ -598,4 +593,4 @@ public class GTAggregateScanner implements IGTScanner { } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java index f1f84af..717f89c 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java @@ -63,11 +63,6 @@ public class GTFilterScanner implements IGTScanner { } @Override - public long getScannedRowCount() { - return inputScanner.getScannedRowCount(); - } - - @Override public void close() throws IOException { inputScanner.close(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 651e5c4..4629c8e 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -42,7 +42,7 @@ import com.google.common.collect.Sets; public class GTScanRequest { private static final Logger logger = LoggerFactory.getLogger(GTScanRequest.class); - + //it's not necessary to increase the checkInterval to very large because the check cost is not high //changing it might break org.apache.kylin.query.ITKylinQueryTest.testTimeoutQuery() public static final int terminateCheckInterval = 100; @@ -175,8 +175,8 @@ public class GTScanRequest { public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, boolean spillEnabled) throws IOException { IGTScanner result = scanner; if (!filterToggledOn) { //Skip reading this section if you're not profiling! - int scanned = lookAndForget(result); - return new EmptyGTScanner(scanned); + lookAndForget(result); + return new EmptyGTScanner(); } else { if (this.hasFilterPushDown() && !hasPreFiltered) { @@ -184,9 +184,8 @@ public class GTScanRequest { } if (!aggrToggledOn) {//Skip reading this section if you're not profiling! - long scanned = result.getScannedRowCount(); lookAndForget(result); - return new EmptyGTScanner(scanned); + return new EmptyGTScanner(); } if (!this.isAllowStorageAggregation()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java index 980787b..96c7972 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java @@ -21,9 +21,5 @@ package org.apache.kylin.gridtable; import java.io.Closeable; public interface IGTScanner extends Iterable<GTRecord>, Closeable { - GTInfo getInfo(); - - long getScannedRowCount(); - } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java index 9c839bb..71a0a21 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java @@ -184,12 +184,6 @@ public class SortedGTRecordGenerator { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return counter; - } - } private class Distribution { http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java index f9f370b..e1b5406 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java @@ -106,11 +106,6 @@ public class GTSimpleMemStore implements IGTStore { } @Override - public long getScannedRowCount() { - return count; - } - - @Override public void close() throws IOException { } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java index 7abb069..8b2243c 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java @@ -68,11 +68,6 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { } @Override - public long getScannedRowCount() { - throw new UnsupportedOperationException(); - } - - @Override public void close() throws IOException { } @@ -109,11 +104,6 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { } @Override - public long getScannedRowCount() { - throw new UnsupportedOperationException(); - } - - @Override public void close() throws IOException { } @@ -141,4 +131,4 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { assertEquals(10, count); scanner.close(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java index 4ac6644..14a25c5 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java @@ -18,7 +18,6 @@ package org.apache.kylin.gridtable; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -53,7 +52,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { GTBuilder builder = rebuild(table); IGTScanner scanner = scan(table); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); } @Test @@ -64,7 +62,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { GTBuilder builder = rebuild(table); IGTScanner scanner = scan(table); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); } @Test @@ -75,7 +72,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { GTBuilder builder = rebuild(table); IGTScanner scanner = scanAndAggregate(table); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); } @Test @@ -86,7 +82,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { rebuildViaAppend(table); IGTScanner scanner = scan(table); - assertEquals(10, scanner.getScannedRowCount()); } private IGTScanner scan(GridTable table) throws IOException { @@ -101,12 +96,11 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { System.out.println(r); } scanner.close(); - System.out.println("Scanned Row Count: " + scanner.getScannedRowCount()); return scanner; } private IGTScanner scanAndAggregate(GridTable table) throws IOException { - GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[]{"count", "sum"}).setFilterPushDown(null).createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[] { "count", "sum" }).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) { @@ -135,7 +129,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { System.out.println(r); } scanner.close(); - System.out.println("Scanned Row Count: " + scanner.getScannedRowCount()); return scanner; } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java index 029502c..4f206d4 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java @@ -96,11 +96,6 @@ public class CubeSegmentScanner implements IGTScanner { return scanRequest == null ? null : scanRequest.getInfo(); } - @Override - public long getScannedRowCount() { - return scanner.getScannedRowCount(); - } - public CubeSegment getSegment() { return this.cubeSeg; } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java index fd50c54..9e89227 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java @@ -18,6 +18,10 @@ package org.apache.kylin.storage.gtrecord; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.Iterator; + import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.gridtable.EmptyGTScanner; import org.apache.kylin.gridtable.GTInfo; @@ -29,10 +33,6 @@ import org.apache.kylin.metadata.model.ISegment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.util.Iterator; - public class ScannerWorker { private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class); @@ -41,7 +41,7 @@ public class ScannerWorker { public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) { if (scanRequest == null) { logger.info("Segment {} will be skipped", segment); - internal = new EmptyGTScanner(0); + internal = new EmptyGTScanner(); return; } @@ -62,9 +62,4 @@ public class ScannerWorker { public void close() throws IOException { internal.close(); } - - public long getScannedRowCount() { - return internal.getScannedRowCount(); - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java index dc8746f..3904b5c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java @@ -45,15 +45,13 @@ public class StorageResponseGTScatter implements IGTScanner { private IPartitionStreamer partitionStreamer; private Iterator<byte[]> blocks; private ImmutableBitSet columns; - private long totalScannedCount; private int storagePushDownLimit = -1; - public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, long totalScannedCount, int storagePushDownLimit) { + public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, int storagePushDownLimit) { this.info = info; this.partitionStreamer = partitionStreamer; this.blocks = partitionStreamer.asByteArrayIterator(); this.columns = columns; - this.totalScannedCount = totalScannedCount; this.storagePushDownLimit = storagePushDownLimit; } @@ -63,11 +61,6 @@ public class StorageResponseGTScatter implements IGTScanner { } @Override - public long getScannedRowCount() { - return totalScannedCount; - } - - @Override public void close() throws IOException { //If upper consumer failed while consuming the GTRecords, the consumer should call IGTScanner's close method to ensure releasing resource partitionStreamer.close(); http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java new file mode 100644 index 0000000..a3720c8 --- /dev/null +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.query; + +import java.io.File; +import java.util.Map; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.QueryContext; +import org.apache.kylin.common.exceptions.ResourceLimitExceededException; +import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.query.routing.Candidate; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import com.google.common.collect.Maps; + +public class ITFailfastQueryTest extends KylinTestBase { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @BeforeClass + public static void setUp() throws Exception { + printInfo("setUp in ITFailfastQueryTest"); + Map<RealizationType, Integer> priorities = Maps.newHashMap(); + priorities.put(RealizationType.HYBRID, 0); + priorities.put(RealizationType.CUBE, 0); + priorities.put(RealizationType.INVERTED_INDEX, 0); + Candidate.setPriorities(priorities); + joinType = "left"; + setupAll(); + } + + @After + public void cleanUp() { + QueryContext.reset(); + } + + @AfterClass + public static void tearDown() throws Exception { + printInfo("tearDown in ITFailfastQueryTest"); + Candidate.restorePriorities(); + clean(); + } + + @Test + public void testPartitionExceedMaxScanBytes() throws Exception { + String key = "kylin.storage.partition.max-scan-bytes"; + long saved = KylinConfig.getInstanceFromEnv().getPartitionMaxScanBytes(); + KylinConfig.getInstanceFromEnv().setProperty(key, "18000");//very low threshold + + boolean meetExpectedException = false; + try { + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql"; + File sqlFile = new File(queryFileName); + try { + runSQL(sqlFile, false, false); + } catch (Exception e) { + if (findRoot(e) instanceof ResourceLimitExceededException) { + //expected + meetExpectedException = true; + } else { + throw new RuntimeException(e); + } + } + + if (!meetExpectedException) { + throw new RuntimeException("Did not meet expected exception"); + } + } finally { + KylinConfig.getInstanceFromEnv().setProperty(key, String.valueOf(saved)); + } + } + + @Test + public void testPartitionNotExceedMaxScanBytes() throws Exception { + String key = "kylin.storage.partition.max-scan-bytes"; + long saved = KylinConfig.getInstanceFromEnv().getPartitionMaxScanBytes(); + KylinConfig.getInstanceFromEnv().setProperty(key, "20000");//enough threshold + + try { + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql"; + File sqlFile = new File(queryFileName); + runSQL(sqlFile, false, false); + } finally { + KylinConfig.getInstanceFromEnv().setProperty(key, String.valueOf(saved)); + } + } + + @Test + public void testQueryExceedMaxScanBytes() throws Exception { + String key = "kylin.query.max-scan-bytes"; + long saved = KylinConfig.getInstanceFromEnv().getQueryMaxScanBytes(); + KylinConfig.getInstanceFromEnv().setProperty(key, "30000");//very low threshold + + boolean meetExpectedException = false; + try { + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql"; + File sqlFile = new File(queryFileName); + try { + runSQL(sqlFile, false, false); + } catch (Exception e) { + if (findRoot(e) instanceof ResourceLimitExceededException) { + //expected + meetExpectedException = true; + } else { + throw new RuntimeException(e); + } + } + + if (!meetExpectedException) { + throw new RuntimeException("Did not meet expected exception"); + } + } finally { + KylinConfig.getInstanceFromEnv().setProperty(key, String.valueOf(saved)); + } + } + + @Test + public void testQueryNotExceedMaxScanBytes() throws Exception { + String key = "kylin.query.max-scan-bytes"; + long saved = KylinConfig.getInstanceFromEnv().getQueryMaxScanBytes(); + KylinConfig.getInstanceFromEnv().setProperty(key, "40000");//enough threshold + + try { + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql"; + File sqlFile = new File(queryFileName); + runSQL(sqlFile, false, false); + } finally { + KylinConfig.getInstanceFromEnv().setProperty(key, String.valueOf(saved)); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 842ce40..1158704 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -74,21 +74,6 @@ public class ITKylinQueryTest extends KylinTestBase { clean(); } - protected String getQueryFolderPrefix() { - return ""; - } - - protected Throwable findRoot(Throwable throwable) { - while (true) { - if (throwable.getCause() != null) { - throwable = throwable.getCause(); - } else { - break; - } - } - return throwable; - } - @Test public void testTimeoutQuery() throws Exception { try { @@ -121,9 +106,6 @@ public class ITKylinQueryTest extends KylinTestBase { try { runSQL(sqlFile, false, false); } catch (SQLException e) { - - System.out.println(e.getMessage()); - if (findRoot(e) instanceof KylinTimeoutException) { //expected continue; http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 402aaa0..fd04b2f 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -692,4 +692,19 @@ public class KylinTestBase { return OLAPContext.getThreadLocalContexts().iterator().next(); } + protected String getQueryFolderPrefix() { + return ""; + } + + protected Throwable findRoot(Throwable throwable) { + while (true) { + if (throwable.getCause() != null) { + throwable = throwable.getCause(); + } else { + break; + } + } + return throwable; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 4c02aa4..122b823 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -361,14 +361,16 @@ public class QueryService extends BasicService { sqlResponse = query(sqlRequest); long durationThreshold = kylinConfig.getQueryDurationCacheThreshold(); - long scancountThreshold = kylinConfig.getQueryScanCountCacheThreshold(); + long scanCountThreshold = kylinConfig.getQueryScanCountCacheThreshold(); + long scanBytesThreshold = kylinConfig.getQueryScanBytesCacheThreshold(); sqlResponse.setDuration(System.currentTimeMillis() - startTime); logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", // String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount())); - if (checkCondition(queryCacheEnabled, "query cache is disabled") && // - checkCondition(!sqlResponse.getIsException(), "query has exception") && // - checkCondition(sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold, "query is too lightweight with duration: {} ({}), scan count: {} ({})", sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(), scancountThreshold) && // - checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) { + if (checkCondition(queryCacheEnabled, "query cache is disabled") // + && checkCondition(!sqlResponse.getIsException(), "query has exception") // + && checkCondition(sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scanCountThreshold || sqlResponse.getTotalScanBytes() > scanBytesThreshold, // + "query is too lightweight with duration: {} (threshold {}), scan count: {} (threshold {}), scan bytes: {} (threshold {})", sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(), scanCountThreshold, sqlResponse.getTotalScanBytes(), scanBytesThreshold) + && checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) { cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java index 3fdb92f..3eecba1 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java @@ -132,9 +132,6 @@ public class HBaseScannerBenchmark { } scan.close(); - if (scan.getScannedRowCount() != count) - throw new IllegalStateException(); - t = System.currentTimeMillis() - t; logger.info(msg + ", " + count + " records, " + speed(t) + "K rec/sec"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java index f63d9c2..b12173d 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java @@ -205,10 +205,5 @@ public class SimpleHBaseStore implements IGTStore { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return count; - } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index abc3437..82b67b6 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -23,7 +23,6 @@ import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.zip.DataFormatException; @@ -146,8 +145,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { logger.debug("Submitting rpc to {} shards starting from shard {}, scan range count {}", shardNum, cuboidBaseShard, rawScans.size()); - final AtomicLong totalScannedCount = new AtomicLong(0); - // KylinConfig: use env instance instead of CubeSegment, because KylinConfig will share among queries // for different cubes until redeployment of coprocessor jar. final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); @@ -205,7 +202,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { Stats stats = result.getStats(); queryContext.addAndGetScannedRows(stats.getScannedRowCount()); queryContext.addAndGetScannedBytes(stats.getScannedBytes()); - totalScannedCount.addAndGet(stats.getScannedRowCount()); // if any other region has responded with error, skip further processing if (regionErrorHolder.get() != null) { @@ -249,7 +245,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { }); } - return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit()); + return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), scanRequest.getStoragePushDownLimit()); } private ByteString serializeGTScanReq(GTScanRequest scanRequest) { @@ -317,14 +313,14 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { CubeVisitResponse.ErrorInfo errorInfo = response.getErrorInfo(); switch (errorInfo.getType()) { - case UNKNOWN_TYPE: - return new RuntimeException("Coprocessor aborts: " + errorInfo.getMessage()); - case TIMEOUT: - return new KylinTimeoutException(errorInfo.getMessage()); - case RESOURCE_LIMIT_EXCEEDED: - return new ResourceLimitExceededException("Coprocessor resource limit exceeded: " + errorInfo.getMessage()); - default: - throw new AssertionError("Unknown error type: " + errorInfo.getType()); + case UNKNOWN_TYPE: + return new RuntimeException("Coprocessor aborts: " + errorInfo.getMessage()); + case TIMEOUT: + return new KylinTimeoutException(errorInfo.getMessage()); + case RESOURCE_LIMIT_EXCEEDED: + return new ResourceLimitExceededException("Coprocessor resource limit exceeded: " + errorInfo.getMessage()); + default: + throw new AssertionError("Unknown error type: " + errorInfo.getType()); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java index 1698180..33f8d90 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java @@ -93,31 +93,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC { @Override public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws IOException { - final IGTScanner scanner = getGTScannerInternal(scanRequest); - - return new IGTScanner() { - @Override - public GTInfo getInfo() { - return scanner.getInfo(); - } - - @Override - public long getScannedRowCount() { - long sum = 0; - sum += scanner.getScannedRowCount(); - return sum; - } - - @Override - public void close() throws IOException { - scanner.close(); - } - - @Override - public Iterator<GTRecord> iterator() { - return scanner.iterator(); - } - }; + return getGTScannerInternal(scanRequest); } //for non-sharding cases it will only return one byte[] with not shard at beginning @@ -229,11 +205,6 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC { } @Override - public long getScannedRowCount() { - return decorateScanner.getScannedRowCount(); - } - - @Override public void close() throws IOException { decorateScanner.close(); } @@ -244,4 +215,4 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC { } }; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java index 4c02dff..631e8e8 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java @@ -152,11 +152,6 @@ public class HBaseReadonlyStore implements IGTStore { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return count; - } }; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java index 61cf067..cde127e 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java @@ -293,7 +293,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement ResourceTrackingCellListIterator cellListIterator = new ResourceTrackingCellListIterator( allCellLists, scanReq.getStorageScanRowNumThreshold(), // for old client (scan threshold) - request.hasMaxScanBytes() ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client + !request.hasMaxScanBytes() ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client scanReq.getTimeout()); IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize(), behavior.delayToggledOn()); http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto index e01ff52..aa83595 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto @@ -37,7 +37,7 @@ message CubeVisitRequest { required string kylinProperties = 5; // kylin properties optional string queryId = 6; optional bool spillEnabled = 7 [default = true]; - optional int64 maxScanBytes = 8; // 0 means no limit + optional int64 maxScanBytes = 8; // must be positive message IntList { repeated int32 ints = 1; } http://git-wip-us.apache.org/repos/asf/kylin/blob/4bdb62cb/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java index 691886b..62b154e 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java @@ -36,7 +36,6 @@ import org.slf4j.LoggerFactory; * It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment */ public class SandboxMetastoreCLI { - private static final Logger logger = LoggerFactory.getLogger(SandboxMetastoreCLI.class); public static void main(String[] args) throws Exception {
