KYLIN-2993 Add special mr config for base cuboid step Signed-off-by: Li Yang <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad3dd3f6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad3dd3f6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad3dd3f6 Branch: refs/heads/master Commit: ad3dd3f63d399640142bd2c97669080bc3e37560 Parents: a79035a Author: kangkaisen <[email protected]> Authored: Mon Sep 4 10:29:00 2017 +0800 Committer: Li Yang <[email protected]> Committed: Thu Dec 14 14:58:17 2017 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../kylin/storage/hbase/steps/HBaseMROutput2Transition.java | 9 +++++++++ 2 files changed, 13 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/ad3dd3f6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index d59c98d..524b1d4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1003,6 +1003,10 @@ abstract public class KylinConfigBase implements Serializable { return getPropertiesByPrefix("kylin.engine.mr.uhc-config-override."); } + public Map<String, String> getBaseCuboidMRConfigOverride() { + return getPropertiesByPrefix("kylin.engine.mr.base-cuboid-config-override."); + } + public Map<String, String> getSparkConfigOverride() { return getPropertiesByPrefix("kylin.engine.spark-conf."); } http://git-wip-us.apache.org/repos/asf/kylin/blob/ad3dd3f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java index db3f7f4..8817cb2 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java @@ -19,6 +19,7 @@ package org.apache.kylin.storage.hbase.steps; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -99,6 +100,14 @@ public class HBaseMROutput2Transition implements IMROutput2 { int level) throws Exception { int reducerNum = 1; Class mapperClass = job.getMapperClass(); + + //allow user specially set config for base cuboid step + if (mapperClass == HiveToBaseCuboidMapper.class) { + for (Map.Entry<String, String> entry : segment.getConfig().getBaseCuboidMRConfigOverride().entrySet()) { + job.getConfiguration().set(entry.getKey(), entry.getValue()); + } + } + if (mapperClass == HiveToBaseCuboidMapper.class || mapperClass == NDCuboidMapper.class) { reducerNum = MapReduceUtil.getLayeredCubingReduceTaskNum(segment, cuboidScheduler, AbstractHadoopJob.getTotalMapInputMB(job), level);
