minor, add cap for reducer number in RedistributeFlatHiveTableStep Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/111e7927 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/111e7927 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/111e7927
Branch: refs/heads/KYLIN-1971 Commit: 111e79275ecf4393d427a17f7f6654d74e2ec1a9 Parents: b4329a6 Author: shaofengshi <shaofeng...@apache.org> Authored: Tue Oct 18 18:41:02 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Thu Oct 20 14:27:52 2016 +0800 ---------------------------------------------------------------------- .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 1 + 1 file changed, 1 insertion(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/111e7927/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index f3fceb1..f536cbb 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -344,6 +344,7 @@ public class HiveMRInput implements IMRInput { int numReducers = Math.round(rowCount / ((float) mapperInputRows)); numReducers = Math.max(1, numReducers); + numReducers = Math.min(numReducers, config.getHadoopJobMaxReducerNumber()); stepLogger.log("total input rows = " + rowCount); stepLogger.log("expected input rows per mapper = " + mapperInputRows);
