review of KYLIN-2436,KYILN-2437,KYLIN-2438 and KYLIN-2443
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f474ac5c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f474ac5c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f474ac5c Branch: refs/heads/master-hbase0.98 Commit: f474ac5c1a34170922e485f53e91b3f224c16386 Parents: eb472a3 Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Feb 15 09:31:24 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Wed Feb 15 09:32:18 2017 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 8 ++++---- .../main/java/org/apache/kylin/common/KylinConfigBase.java | 6 +++--- .../main/resources/kylin-backward-compatibility.properties | 2 +- .../kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 2 +- .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 095a53f..f2ebb80 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -88,11 +88,11 @@ kylin.storage.hbase.coprocessor-mem-gb=3 # By default kylin can spill query's intermediate results to disks when it's consuming too much memory. # Set it to false if you want query to abort immediately in such condition. -kylin.storage.hbase.coprocessor-spill-enabled=true +kylin.storage.partition.aggr-spill-enabled=true # The maximum number of bytes each coprocessor is allowed to scan. # To allow arbitrary large scan, you can set it to 0. -kylin.storage.hbase.coprocessor-max-scan-bytes=3221225472 +kylin.storage.partition.max-scan-bytes=3221225472 # The default coprocessor timeout is (hbase.rpc.timeout * 0.9) / 1000 seconds, # You can set it to a smaller value. 0 means use default. @@ -158,7 +158,7 @@ kylin.snapshot.max-mb=300 # Controls the maximum number of bytes a query is allowed to scan storage. # The default value 0 means no limit. -# The counterpart kylin.storage.hbase.coprocessor-max-scan-bytes sets the maximum per coprocessor. +# The counterpart kylin.storage.partition.max-scan-bytes sets the maximum per coprocessor. kylin.query.max-scan-bytes=0 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF @@ -230,4 +230,4 @@ kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history ## manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime #kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar -#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec \ No newline at end of file +#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index fafa96d..9c54d68 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -614,11 +614,11 @@ abstract public class KylinConfigBase implements Serializable { } public boolean getQueryCoprocessorSpillEnabled() { - return Boolean.parseBoolean(this.getOptional("kylin.storage.hbase.coprocessor-spill-enabled", "true")); + return Boolean.parseBoolean(this.getOptional("kylin.storage.partition.aggr-spill-enabled", "true")); } - public long getQueryCoprocessorMaxScanBytes() { - long value = Long.parseLong(this.getOptional("kylin.storage.hbase.coprocessor-max-scan-bytes", String.valueOf(3L * 1024 * 1024 * 1024))); + public long getPartitionMaxScanBytes() { + long value = Long.parseLong(this.getOptional("kylin.storage.partition.max-scan-bytes", String.valueOf(3L * 1024 * 1024 * 1024))); return value > 0 ? value : Long.MAX_VALUE; } http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/core-common/src/main/resources/kylin-backward-compatibility.properties ---------------------------------------------------------------------- diff --git a/core-common/src/main/resources/kylin-backward-compatibility.properties b/core-common/src/main/resources/kylin-backward-compatibility.properties index ab9bcb1..c7d9cd7 100644 --- a/core-common/src/main/resources/kylin-backward-compatibility.properties +++ b/core-common/src/main/resources/kylin-backward-compatibility.properties @@ -152,7 +152,7 @@ kylin.query.transformers=kylin.query.transformers kylin.query.cache.enabled=kylin.query.cache-enabled kylin.query.cache.threshold.duration=kylin.query.cache-threshold-duration kylin.query.cache.threshold.scancount=kylin.query.cache-threshold-scan-count -kylin.query.mem.budget=kylin.storage.hbase.coprocessor-max-scan-bytes +kylin.query.mem.budget=kylin.storage.partition.max-scan-bytes kylin.query.ignore_unknown_function=kylin.query.ignore-unknown-function kylin.query.dim.distinct.max=kylin.query.max-dimension-count-distinct kylin.query.security.enabled=kylin.query.security-enabled http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index 3c01da2..abc3437 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -164,7 +164,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { builder.setQueryId(queryId); } builder.setSpillEnabled(cubeSeg.getConfig().getQueryCoprocessorSpillEnabled()); - builder.setMaxScanBytes(cubeSeg.getConfig().getQueryCoprocessorMaxScanBytes()); + builder.setMaxScanBytes(cubeSeg.getConfig().getPartitionMaxScanBytes()); for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) { executorService.submit(new Runnable() { http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java index be89caa..61cf067 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java @@ -293,7 +293,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement ResourceTrackingCellListIterator cellListIterator = new ResourceTrackingCellListIterator( allCellLists, scanReq.getStorageScanRowNumThreshold(), // for old client (scan threshold) - request.getMaxScanBytes() == 0 ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client + request.hasMaxScanBytes() ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client scanReq.getTimeout()); IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize(), behavior.delayToggledOn());
