Repository: kylin Updated Branches: refs/heads/pr70 f0bc691ec -> 7834b7963
KYLIN-2802 minor code review Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7834b796 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7834b796 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7834b796 Branch: refs/heads/pr70 Commit: 7834b7963d61ef34d70db3c7a816f4cee0c2576e Parents: f0bc691 Author: Li Yang <liy...@apache.org> Authored: Sat Sep 23 18:53:48 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sat Sep 23 18:53:48 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 59 +++++++++++--------- .../main/resources/kylin-defaults.properties | 9 +++ .../org/apache/kylin/cube/CubeInstance.java | 30 ---------- .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../kylin/engine/mr/common/CubeStatsReader.java | 2 +- .../engine/mr/common/CuboidStatsReaderUtil.java | 2 +- .../mr/common/StatisticsDecisionUtil.java | 4 ++ 7 files changed, 48 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7834b796/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index eb2adab..71de284 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -409,6 +409,38 @@ abstract public class KylinConfigBase implements Serializable { } // ============================================================================ + // Cube Planner + // ============================================================================ + + public boolean isCubePlannerEnabled() { + return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled", "false")); + } + + public boolean isCubePlannerEnabledForExistingCube() { + return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube", "false")); + } + + public double getCubePlannerExpansionRateThreshold() { + return Double.parseDouble(getOptional("kylin.cube.cubeplanner.expansion-threshold", "15.0")); + } + + public int getCubePlannerRecommendCuboidCacheMaxSize() { + return Integer.parseInt(getOptional("kylin.cube.cubeplanner.recommend-cache-max-size", "200")); + } + + public long getCubePlannerMandatoryRollUpThreshold() { + return Long.parseLong(getOptional("kylin.cube.cubeplanner.mandatory-rollup-threshold", "1000")); + } + + public int getCubePlannerAgreedyAlgorithmAutoThreshold() { + return Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-greedy", "10")); + } + + public int getCubePlannerGeneticAlgorithmAutoThreshold() { + return Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-genetic", "23")); + } + + // ============================================================================ // JOB // ============================================================================ @@ -1224,33 +1256,6 @@ abstract public class KylinConfigBase implements Serializable { } // ============================================================================ - // Cube Planner - // ============================================================================ - public boolean isCubePlannerEnabledForExistingCube() { - return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube", "false")); - } - - public double getCubePlannerExpansionRateThreshold() { - return Double.parseDouble(getOptional("kylin.cube.cubeplanner.expansion-threshold", "15.0")); - } - - public int getCubePlannerRecommendCuboidCacheMaxSize() { - return Integer.parseInt(getOptional("kylin.cube.cubeplanner.recommend-cache-max-size", "200")); - } - - public long getCubePlannerMandatoryRollUpThreshold() { - return Long.parseLong(getOptional("kylin.cube.cubeplanner.mandatory-rollup-threshold", "1000")); - } - - public int getCubePlannerAgreedyAlgorithmAutoThreshold() { - return Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-greedy", "10")); - } - - public int getCubePlannerGeneticAlgorithmAutoThreshold() { - return Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-genetic", "23")); - } - - // ============================================================================ // RESTCLIENT // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/7834b796/core-common/src/main/resources/kylin-defaults.properties ---------------------------------------------------------------------- diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties index c2da3f8..ad313ca 100644 --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@ -157,6 +157,7 @@ kylin.engine.mr.build-dict-in-reducer=true # Number of reducers for fetching UHC column distinct values kylin.engine.mr.uhc-reducer-count=1 + ### CUBE | DICTIONARY ### kylin.cube.cuboid-scheduler=org.apache.kylin.cube.cuboid.DefaultCuboidScheduler @@ -172,6 +173,14 @@ kylin.cube.aggrgroup.max-combination=4096 kylin.snapshot.max-mb=300 +kylin.cube.cubeplanner.enabled=false +kylin.cube.cubeplanner.enabled-for-existing-cube=false +kylin.cube.cubeplanner.expansion-threshold=15.0 +kylin.cube.cubeplanner.recommend-cache-max-size=200 +kylin.cube.cubeplanner.mandatory-rollup-threshold=1000 +kylin.cube.cubeplanner.algorithm-threshold-greedy=10 +kylin.cube.cubeplanner.algorithm-threshold-genetic=23 + ### QUERY ### http://git-wip-us.apache.org/repos/asf/kylin/blob/7834b796/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index a611b96..5177898 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -29,7 +29,6 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.CompressionUtils; import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.cube.cuboid.CuboidModeEnum; import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.cuboid.TreeCuboidScheduler; import org.apache.kylin.cube.model.CubeDesc; @@ -328,35 +327,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, this.createTimeUTC = createTimeUTC; } - public Set<Long> getCuboidsByMode(String cuboidModeName) { - return getCuboidsByMode(CuboidModeEnum.getByModeName(cuboidModeName)); - } - - public Set<Long> getCuboidsByMode(CuboidModeEnum cuboidMode) { - if (cuboidMode == null || cuboidMode == CuboidModeEnum.CURRENT) { - return getCuboidScheduler().getAllCuboidIds(); - } - Set<Long> cuboidsRecommend = getCuboidsRecommend(); - if (cuboidsRecommend == null || cuboidMode == CuboidModeEnum.RECOMMEND) { - return cuboidsRecommend; - } - Set<Long> currentCuboids = getCuboidScheduler().getAllCuboidIds(); - switch (cuboidMode) { - case RECOMMEND_EXISTING: - cuboidsRecommend.retainAll(currentCuboids); - return cuboidsRecommend; - case RECOMMEND_MISSING: - cuboidsRecommend.removeAll(currentCuboids); - return cuboidsRecommend; - case RECOMMEND_MISSING_WITH_BASE: - cuboidsRecommend.removeAll(currentCuboids); - currentCuboids.add(getCuboidScheduler().getBaseCuboidId()); - return cuboidsRecommend; - default: - return null; - } - } - public Map<Long, Long> getCuboids() { if (cuboidBytes == null) return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/7834b796/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index d4d88fd..0ba7f21 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -418,7 +418,7 @@ public class CubeManager implements IRealizationProvider { cube.setCost(update.getCost()); } - if(update.getCuboids() != null){ + if (update.getCuboids() != null) { cube.setCuboids(update.getCuboids()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/7834b796/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java index 8f1da6e..c82d797 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java @@ -140,7 +140,7 @@ public class CubeStatsReader { return tempFile; } - public Map<Long, HLLCounter> getCuboidRowEstimatesHLLOrigin() { + public Map<Long, HLLCounter> getCuboidRowHLLCounters() { return this.cuboidRowEstimatesHLL; } http://git-wip-us.apache.org/repos/asf/kylin/blob/7834b796/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java index 3025750..68380f3 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java @@ -75,7 +75,7 @@ public class CuboidStatsReaderUtil { Map<Long, Double> sizeMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size()); for (CubeSegment pSegment : segmentList) { CubeStatsReader pReader = new CubeStatsReader(pSegment, pSegment.getConfig()); - Map<Long, HLLCounter> pHLLMap = pReader.getCuboidRowEstimatesHLLOrigin(); + Map<Long, HLLCounter> pHLLMap = pReader.getCuboidRowHLLCounters(); if (pHLLMap == null || pHLLMap.isEmpty()) { logger.info("Cuboid Statistics for segment " + pSegment.getName() + " is not enabled."); nSegment--; http://git-wip-us.apache.org/repos/asf/kylin/blob/7834b796/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java index 4145226..9c805a8 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java @@ -90,6 +90,10 @@ public class StatisticsDecisionUtil { public static void optimizeCubingPlan(CubeSegment segment) throws IOException { CubeInstance cube = segment.getCubeInstance(); + + if (cube.getConfig().isCubePlannerEnabled() == false) + return; + List<CubeSegment> readySegments = cube.getSegments(SegmentStatusEnum.READY); if (readySegments.size() == 0 || (cube.getConfig().isCubePlannerEnabledForExistingCube() && readySegments.size() == 1 && (readySegments.get(0).getSegRange().equals(segment.getSegRange())))) {
