KYLIN-2866 minor refine
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7a32458 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7a32458 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7a32458 Branch: refs/heads/master Commit: e7a32458c9f8b82345c029200db2658298a12815 Parents: ebafc76 Author: lidongsjtu <[email protected]> Authored: Sun Dec 17 16:59:12 2017 +0800 Committer: lidongsjtu <[email protected]> Committed: Wed Dec 20 23:20:11 2017 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/common/KylinConfigBase.java | 9 +++++---- .../main/java/org/apache/kylin/common/util/HadoopUtil.java | 4 ++-- .../kylin/engine/mr/steps/MergeStatisticsWithOldStep.java | 2 +- .../apache/kylin/engine/mr/steps/SaveStatisticsStep.java | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e7a32458/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 23a2120..1b3aa03 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1040,12 +1040,13 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.engine.mr.cuboid-number-per-stats-calculator", "100")); } - public int getFactDistinctJobPerReducerHLLCuboidNumber() { - return Integer.parseInt(getOptional("kylin.engine.mr.fact-distinct-per-reducer-hll-cuboid-number", "100")); + public int getHadoopJobPerReducerHLLCuboidNumber() { + return Integer.parseInt(getOptional("kylin.engine.mr.per-reducer-hll-cuboid-number", "100")); } - public int getFactDistinctJobHLLMaxReducerNumber() { - return Integer.parseInt(getOptional("kylin.engine.mr.fact-distinct-hll-max-reducer-number", "50")); + public int getHadoopJobHLLMaxReducerNumber() { + // by default multi-reducer hll calculation is disabled + return Integer.parseInt(getOptional("kylin.engine.mr.hll-max-reducer-number", "1")); } //UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns http://git-wip-us.apache.org/repos/asf/kylin/blob/e7a32458/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java index cafcaf2..64b9f46 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java @@ -161,7 +161,7 @@ public class HadoopUtil { } } - public static Path[] getFilterPath(FileSystem fs, Path baseDir, final String filter) throws IOException { + public static Path[] getFilteredPath(FileSystem fs, Path baseDir, final String prefix) throws IOException { if (fs.exists(baseDir) == false) { return null; } @@ -169,7 +169,7 @@ public class HadoopUtil { FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() { @Override public boolean accept(Path path) { - return path.getName().startsWith(filter); + return path.getName().startsWith(prefix); } }); http://git-wip-us.apache.org/repos/asf/kylin/blob/e7a32458/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java index 7855c06..eca0499 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java @@ -85,7 +85,7 @@ public class MergeStatisticsWithOldStep extends AbstractExecutable { throw new IOException("StatisticsFilePath " + statisticsDirPath + " is not a directory"); } - Path[] statisticsFiles = HadoopUtil.getFilterPath(hdfs, statisticsDirPath, + Path[] statisticsFiles = HadoopUtil.getFilteredPath(hdfs, statisticsDirPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME); if (statisticsFiles == null) { throw new IOException("fail to find the statistics file in base dir: " + statisticsDirPath); http://git-wip-us.apache.org/repos/asf/kylin/blob/e7a32458/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java index 99ebbef..cbf705c 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java @@ -70,7 +70,7 @@ public class SaveStatisticsStep extends AbstractExecutable { FileSystem fs = HadoopUtil.getWorkingFileSystem(); Configuration hadoopConf = HadoopUtil.getCurrentConfiguration(); Path statisticsDir = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams())); - Path[] statisticsFiles = HadoopUtil.getFilterPath(fs, statisticsDir, BatchConstants.CFG_OUTPUT_STATISTICS); + Path[] statisticsFiles = HadoopUtil.getFilteredPath(fs, statisticsDir, BatchConstants.CFG_OUTPUT_STATISTICS); if (statisticsFiles == null) { throw new IOException("fail to find the statistics file in base dir: " + statisticsDir); } @@ -146,7 +146,7 @@ public class SaveStatisticsStep extends AbstractExecutable { private void logMapperAndCuboidStatistics(Map<Long, HLLCounter> cuboidHLLMap, int samplingPercentage, int mapperNumber, long grantTotal, long totalRowsBeforeMerge) throws IOException { logger.debug("Total cuboid number: \t" + cuboidHLLMap.size()); - logger.debug("Samping percentage: \t" + samplingPercentage); + logger.debug("Sampling percentage: \t" + samplingPercentage); logger.debug("The following statistics are collected based on sampling data."); logger.debug("Number of Mappers: " + mapperNumber);
