KYLIN-2357 Make ERROR_RECORD_LOG_THRESHOLD configurable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8d374f84 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8d374f84 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8d374f84 Branch: refs/heads/master-hbase1.x Commit: 8d374f84368194d78208ea3b7683a697e8ec4314 Parents: eb18ac5 Author: kangkaisen <kangkai...@live.com> Authored: Sat Jan 7 19:16:30 2017 +0800 Committer: kangkaisen <kangkai...@163.com> Committed: Sat Jan 14 19:14:42 2017 +0800 ---------------------------------------------------------------------- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../java/org/apache/kylin/engine/mr/common/BatchConstants.java | 1 - .../org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java | 2 +- .../kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 494c641..d6774ff 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -423,6 +423,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0")); } + public Integer getErrorRecordThreshold() { + return Integer.parseInt(getOptional("kylin.job.error-record-threshold", "0")); + } + // ============================================================================ // SOURCE.HIVE // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java index 078d80f..0281539 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java @@ -80,7 +80,6 @@ public interface BatchConstants { */ String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder"; int NORMAL_RECORD_LOG_THRESHOLD = 100000; - int ERROR_RECORD_LOG_THRESHOLD = 100; /** * dictionaries builder class http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java index d08e29a..93e413b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java @@ -87,7 +87,7 @@ abstract public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<K // TODO expose errorRecordCounter as hadoop counter errorRecordCounter++; - if (errorRecordCounter > BatchConstants.ERROR_RECORD_LOG_THRESHOLD) { + if (errorRecordCounter > cubeSegment.getConfig().getErrorRecordThreshold()) { if (ex instanceof IOException) throw (IOException) ex; else if (ex instanceof RuntimeException) http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java index 74aebb0..34624e3 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java @@ -105,7 +105,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends Kyli ex.printStackTrace(System.err); errorRecordCounter++; - if (errorRecordCounter > BatchConstants.ERROR_RECORD_LOG_THRESHOLD) { + if (errorRecordCounter > cubeSeg.getConfig().getErrorRecordThreshold()) { if (ex instanceof IOException) throw (IOException) ex; else if (ex instanceof RuntimeException)
