Repository: kylin Updated Branches: refs/heads/helix-201602 8aef3cab0 -> 3b4c9928a
Revert "KYLIN-1415 Cube parallel merge" This reverts commit 8aef3cab02bc670dbabb7d1418b8e3a8a394f8fc. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3b4c9928 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3b4c9928 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3b4c9928 Branch: refs/heads/helix-201602 Commit: 3b4c9928ac18c1db0929afcf469d3135d5c7c7a5 Parents: 8aef3ca Author: shaofengshi <shaofeng...@apache.org> Authored: Tue Feb 16 10:40:07 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Feb 16 10:40:07 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/cube/CubeManager.java | 54 ++++++++------------ .../apache/kylin/rest/service/CubeService.java | 2 +- 2 files changed, 23 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3b4c9928/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index aac2701..84dd30a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -18,11 +18,13 @@ package org.apache.kylin.cube; -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.Nullable; + import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; @@ -45,17 +47,21 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.ProjectManager; -import org.apache.kylin.metadata.realization.*; +import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.metadata.realization.IRealizationProvider; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; +import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.source.ReadableTable; import org.apache.kylin.source.SourceFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; /** * @author yangli9 @@ -452,14 +458,8 @@ public class CubeManager implements IRealizationProvider { } public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg) throws IOException { - return mergeSegments(cube, startDate, endDate, forceMergeEmptySeg, true); - } - - public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg, boolean strictCheck) throws IOException { + checkNoBuildingSegment(cube); checkCubeIsPartitioned(cube); - - if (strictCheck) - checkNoBuildingSegment(cube); Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate); CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond()); @@ -619,23 +619,13 @@ public class CubeManager implements IRealizationProvider { return null; } - List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY)); - - if (readySegments.size() == 0) { - logger.debug("Cube " + cube.getName() + " has no ready segment to merge"); + if (cube.getBuildingSegments().size() > 0) { + logger.debug("Cube " + cube.getName() + " has bulding segment, will not trigger merge at this moment"); return null; } - List<CubeSegment> buildingSegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.NEW)); - List<CubeSegment> toSkipSegments = Lists.newArrayList(); - for (CubeSegment building : buildingSegments) { - for (CubeSegment ready : readySegments) { - if (ready.getDateRangeStart() >= building.getDateRangeStart() && ready.getDateRangeEnd() <= building.getDateRangeEnd()) { - toSkipSegments.add(ready); - } - } - } - readySegments.removeAll(toSkipSegments); + List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY)); + if (readySegments.size() == 0) { logger.debug("Cube " + cube.getName() + " has no ready segment to merge"); return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/3b4c9928/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index 552ab4a..066f0c7 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -633,7 +633,7 @@ public class CubeService extends BasicService { cube = getCubeManager().getCube(cubeName); CubeSegment newSeg = getCubeManager().autoMergeCubeSegments(cube); if (newSeg != null) { - newSeg = getCubeManager().mergeSegments(cube, newSeg.getDateRangeStart(), newSeg.getDateRangeEnd(), true, false); + newSeg = getCubeManager().mergeSegments(cube, newSeg.getDateRangeStart(), newSeg.getDateRangeEnd(), true); logger.debug("Will submit merge job on " + newSeg); DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM"); getExecutableManager().addJob(job);
