KYLIN-1601 Need not to shrink scan cache when hbase rows can be large
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/abe590a2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/abe590a2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/abe590a2 Branch: refs/heads/master Commit: abe590a20e5f93d6eb5d598fe117195b1f4d7fbb Parents: 37d63f4 Author: Hongbin Ma <mahong...@apache.org> Authored: Thu Apr 21 18:11:15 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Apr 21 18:19:30 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/storage/hbase/HBaseResourceStore.java | 4 +++- .../kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java | 6 +++--- .../org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/abe590a2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index bb5382f..2262482 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -148,7 +148,9 @@ public class HBaseResourceStore extends ResourceStore { private void tuneScanParameters(Scan scan) { // divide by 10 as some resource like dictionary or snapshot can be very large - scan.setCaching(kylinConfig.getHBaseScanCacheRows() / 10); + // scan.setCaching(kylinConfig.getHBaseScanCacheRows() / 10); + scan.setCaching(kylinConfig.getHBaseScanCacheRows()); + scan.setMaxResultSize(kylinConfig.getHBaseScanMaxResultSize()); scan.setCacheBlocks(true); } http://git-wip-us.apache.org/repos/asf/kylin/blob/abe590a2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java index d7ea2a0..9eb05d2 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java @@ -258,9 +258,9 @@ public class CubeSegmentTupleIterator implements ITupleIterator { scan.setCacheBlocks(true); // cache less when there are memory hungry measures - if (RowValueDecoder.hasMemHungryMeasures(rowValueDecoders)) { - scan.setCaching(scan.getCaching() / 10); - } +// if (RowValueDecoder.hasMemHungryMeasures(rowValueDecoders)) { +// scan.setCaching(scan.getCaching() / 10); +// } } private void applyFuzzyFilter(Scan scan, HBaseKeyRange keyRange) { http://git-wip-us.apache.org/repos/asf/kylin/blob/abe590a2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java index 49e8593..af5d4b7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java @@ -113,8 +113,8 @@ public abstract class CubeHBaseRPC implements IGTStorage { KylinConfig config = cubeSeg.getCubeDesc().getConfig(); int hbaseCaching = config.getHBaseScanCacheRows(); int hbaseMaxResultSize = config.getHBaseScanMaxResultSize(); - if (isMemoryHungry(selectedColBlocks)) - hbaseCaching /= 10; +// if (isMemoryHungry(selectedColBlocks)) +// hbaseCaching /= 10; return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, hbaseCaching, hbaseMaxResultSize); }
