Repository: kylin Updated Branches: refs/heads/v1.6.0-rc2 783d39917 -> 33bb7c293
KYLIN-2221 rethink on KYLIN-1684 Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/33bb7c29 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/33bb7c29 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/33bb7c29 Branch: refs/heads/v1.6.0-rc2 Commit: 33bb7c293982a052a8c436820c708a038878e30f Parents: 783d399 Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Nov 22 13:42:46 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Nov 22 15:59:17 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 5 ++++ .../gtrecord/GTCubeStorageQueryBase.java | 26 ++++---------------- 2 files changed, 10 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/33bb7c29/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 011155f..c2595fb 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -477,6 +477,11 @@ abstract public class KylinConfigBase implements Serializable { return Float.parseFloat(getOptional("kylin.hbase.hfile.size.gb", "2.0")); } + //check KYLIN-1684, in most cases keep the default value + public boolean isSkippingEmptySegments() { + return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true")); + } + public int getStoragePushDownLimitMax() { return Integer.parseInt(getOptional("kylin.query.pushdown.limit.max", "10000")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/33bb7c29/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 76a8c7b..5e8af88 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -122,24 +122,12 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { List<CubeSegmentScanner> scanners = Lists.newArrayList(); for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) { CubeSegmentScanner scanner; - if (cubeSeg.getInputRecords() == 0) { - if (!skipZeroInputSegment(cubeSeg)) { - logger.warn("cube segment {} input record is 0, " + "it may caused by kylin failed to get the job counter " + "as the hadoop history server wasn't running", cubeSeg); - } else { - logger.warn("cube segment {} input record is 0, skip it ", cubeSeg); - continue; - } - } - try { - scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context, getGTStorage()); - } catch (IllegalArgumentException ex) { - // ref KYLIN-1967, real empty segment can trigger dictionary exception -- IllegalArgumentException: Value not exists! - if (cubeSeg.getInputRecords() == 0) { - logger.warn("cube segment {} input record is 0, skip it still", cubeSeg); - continue; - } - throw ex; + if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) { + logger.info("Skip cube segment {} because its input record is 0", cubeSeg); + continue; } + + scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context, getGTStorage()); scanners.add(scanner); } @@ -149,10 +137,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context); } - protected boolean skipZeroInputSegment(CubeSegment cubeSegment) { - return false; - } - protected abstract String getGTStorage(); private void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
