minor, adjust config for CI sandbox, to avoid too many reducers in CreateHFile step
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74ce81c6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74ce81c6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74ce81c6 Branch: refs/heads/stream_m1 Commit: 74ce81c629bc3f6e6e0487ad0a1353479eb4fb2e Parents: baac10b Author: Li Yang <liy...@apache.org> Authored: Fri Jun 17 10:50:25 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri Jun 17 10:50:25 2016 +0800 ---------------------------------------------------------------------- examples/test_case_data/sandbox/kylin.properties | 6 ++++-- .../org/apache/kylin/storage/hbase/steps/CreateHTableJob.java | 5 ----- 2 files changed, 4 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/74ce81c6/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 3751fe2..410bd74 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -64,12 +64,14 @@ kylin.job.concurrent.max.limit=10 # Max reducer number kylin.job.mapreduce.max.reducer.number=5 -#the percentage of the sampling, default 100% +# The percentage of the sampling, default 100% kylin.job.cubing.inmem.sampling.percent=100 # The cut size for hbase region, in GB. # E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default -kylin.hbase.region.cut=5 +kylin.hbase.region.cut=0.1 +kylin.hbase.region.count.max=5 + # The hfile size of GB, smaller hfile leading to the converting hfile MR has more reducers and be faster # set 0 to disable this optimization kylin.hbase.hfile.size.gb=2 http://git-wip-us.apache.org/repos/asf/kylin/blob/74ce81c6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java index c5276a3..2543e9b 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java @@ -174,11 +174,6 @@ public class CreateHTableJob extends AbstractHadoopJob { Collections.sort(allCuboids); int nRegion = Math.round((float) (totalSizeInM / (cut * 1024L))); - if (kylinConfig.isDevEnv()) { - // for regression test in sandbox - nRegion = Math.round((float) (totalSizeInM / cut)); - } - nRegion = Math.max(kylinConfig.getHBaseRegionCountMin(), nRegion); nRegion = Math.min(kylinConfig.getHBaseRegionCountMax(), nRegion);
