Repository: kylin Updated Branches: refs/heads/master a8f35cf26 -> 2309618b9
APACHE-KYLIN-2866: refine reading statistics file by using CubeStatsReader.CubeStatsResult Signed-off-by: lidongsjtu <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc701662 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc701662 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc701662 Branch: refs/heads/master Commit: cc701662b1a97939b595bf04156bbf6033e85fc2 Parents: 8690fd2 Author: Zhong <[email protected]> Authored: Wed Dec 20 11:13:02 2017 +0800 Committer: lidongsjtu <[email protected]> Committed: Wed Dec 20 23:20:11 2017 +0800 ---------------------------------------------------------------------- .../kylin/engine/mr/common/CubeStatsReader.java | 18 +++++++++--------- .../kylin/engine/mr/steps/SaveStatisticsStep.java | 8 ++++++-- 2 files changed, 15 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/cc701662/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java index d06c22b..8b9b928 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java @@ -98,10 +98,10 @@ public class CubeStatsReader { this.seg = cubeSegment; this.cuboidScheduler = cuboidScheduler; - this.samplingPercentage = cubeStatsResult.percentage; - this.mapperNumberOfFirstBuild = cubeStatsResult.mapperNumber; - this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.mapperOverlapRatio; - this.cuboidRowEstimatesHLL = cubeStatsResult.counterMap; + this.samplingPercentage = cubeStatsResult.getPercentage(); + this.mapperNumberOfFirstBuild = cubeStatsResult.getMapperNumber(); + this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.getMapperOverlapRatio(); + this.cuboidRowEstimatesHLL = cubeStatsResult.getCounterMap(); } /** @@ -120,10 +120,10 @@ public class CubeStatsReader { this.seg = cubeSegment; this.cuboidScheduler = cuboidScheduler; - this.samplingPercentage = cubeStatsResult.percentage; - this.mapperNumberOfFirstBuild = cubeStatsResult.mapperNumber; - this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.mapperOverlapRatio; - this.cuboidRowEstimatesHLL = cubeStatsResult.counterMap; + this.samplingPercentage = cubeStatsResult.getPercentage(); + this.mapperNumberOfFirstBuild = cubeStatsResult.getMapperNumber(); + this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.getMapperOverlapRatio(); + this.cuboidRowEstimatesHLL = cubeStatsResult.getCounterMap(); } private File writeTmpSeqFile(InputStream inputStream) throws IOException { @@ -333,7 +333,7 @@ public class CubeStatsReader { private int percentage = 100; private double mapperOverlapRatio = 0; private int mapperNumber = 0; - Map<Long, HLLCounter> counterMap = Maps.newHashMap(); + private Map<Long, HLLCounter> counterMap = Maps.newHashMap(); public CubeStatsResult(Path path, int precision) throws IOException { Configuration hadoopConf = HadoopUtil.getCurrentConfiguration(); http://git-wip-us.apache.org/repos/asf/kylin/blob/cc701662/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java index cbf705c..921494f 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java @@ -61,7 +61,9 @@ public class SaveStatisticsStep extends AbstractExecutable { @Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { - CubeSegment newSegment = CubingExecutableUtil.findSegment(context, CubingExecutableUtil.getCubeName(this.getParams()), CubingExecutableUtil.getSegmentId(this.getParams())); + CubeSegment newSegment = CubingExecutableUtil.findSegment(context, + CubingExecutableUtil.getCubeName(this.getParams()), + CubingExecutableUtil.getSegmentId(this.getParams())); KylinConfig kylinConf = newSegment.getConfig(); ResourceStore rs = ResourceStore.getStore(kylinConf); @@ -70,7 +72,8 @@ public class SaveStatisticsStep extends AbstractExecutable { FileSystem fs = HadoopUtil.getWorkingFileSystem(); Configuration hadoopConf = HadoopUtil.getCurrentConfiguration(); Path statisticsDir = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams())); - Path[] statisticsFiles = HadoopUtil.getFilteredPath(fs, statisticsDir, BatchConstants.CFG_OUTPUT_STATISTICS); + Path[] statisticsFiles = HadoopUtil.getFilteredPath(fs, statisticsDir, + BatchConstants.CFG_OUTPUT_STATISTICS); if (statisticsFiles == null) { throw new IOException("fail to find the statistics file in base dir: " + statisticsDir); } @@ -83,6 +86,7 @@ public class SaveStatisticsStep extends AbstractExecutable { for (Path item : statisticsFiles) { CubeStatsReader.CubeStatsResult cubeStatsResult = new CubeStatsReader.CubeStatsResult(item, kylinConf.getCubeStatsHLLPrecision()); + cuboidHLLMap.putAll(cubeStatsResult.getCounterMap()); long pGrantTotal = 0L; for (HLLCounter hll : cubeStatsResult.getCounterMap().values()) { pGrantTotal += hll.getCountEstimate();
