KYLIN-1190 Make memory budget per query configurable Signed-off-by: Li, Yang <yang...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7888c87f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7888c87f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7888c87f Branch: refs/heads/1.x-HBase1.1.3 Commit: 7888c87fa190e950227eee80d59f8a108b93a18f Parents: e280002 Author: lidongsjtu <don...@ebay.com> Authored: Tue Dec 1 16:32:17 2015 +0800 Committer: Li, Yang <yang...@ebay.com> Committed: Wed Dec 2 11:11:59 2015 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/common/KylinConfig.java | 4 ++++ .../org/apache/kylin/storage/hbase/CubeStorageEngine.java | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7888c87f/common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java index a8f7c5d..9645b81 100644 --- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -492,6 +492,10 @@ public class KylinConfig { return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true")); } + public long getQueryMemBudget() { + return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024))); + } + public int getHBaseKeyValueSize() { return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/7888c87f/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java index ed12781..bbf088e 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java @@ -76,7 +76,6 @@ public class CubeStorageEngine implements IStorageEngine { private static final Logger logger = LoggerFactory.getLogger(CubeStorageEngine.class); private static final int MERGE_KEYRANGE_THRESHOLD = 100; - private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G private final CubeInstance cubeInstance; private final CubeDesc cubeDesc; @@ -643,8 +642,13 @@ public class CubeStorageEngine implements IStorageEngine { } } - long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst; - context.setThreshold((int) rowEst); + long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst; + if (rowEst > 0) { + logger.info("Memory budget is set to: " + rowEst); + context.setThreshold((int) rowEst); + } else { + logger.info("Memory budget is not set."); + } } private void setLimit(TupleFilter filter, StorageContext context) {
