KYLIN-2154 source table rows will be skipped if TOPN's group column contains NULL values
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9adf10d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9adf10d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9adf10d Branch: refs/heads/KYLIN-1971 Commit: f9adf10d2574bb244e1cbd18bf0f5185e044213b Parents: e3e9ad3 Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Nov 2 22:28:35 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Nov 3 13:34:12 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 9 +++++++-- .../apache/kylin/storage/gtrecord/CubeSegmentScanner.java | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f9adf10d/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java index 7e788e4..f34c57c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java @@ -18,6 +18,7 @@ package org.apache.kylin.measure.topn; +import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -142,8 +143,12 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { final ByteArray key = new ByteArray(keyLength); int offset = 0; for (int i = 0; i < dimensionEncodings.length; i++) { - byte[] valueBytes = Bytes.toBytes(values[i + 1]); - dimensionEncodings[i].encode(valueBytes, valueBytes.length, key.array(), offset); + if (values[i + 1] == null) { + Arrays.fill(key.array(), offset, offset + dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL); + } else { + byte[] valueBytes = Bytes.toBytes(values[i + 1]); + dimensionEncodings[i].encode(valueBytes, valueBytes.length, key.array(), offset); + } offset += dimensionEncodings[i].getLengthOfEncoding(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f9adf10d/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java index 6e19c09..c6a6daa 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java @@ -53,6 +53,9 @@ public class CubeSegmentScanner implements IGTScanner { public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, // Collection<FunctionDesc> metrics, TupleFilter originalfilter, StorageContext context, String gtStorage) { + + logger.info("Init CubeSegmentScanner for segment {}", cubeSeg.getName()); + this.cuboid = cuboid; this.cubeSeg = cubeSeg; @@ -61,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner { //is working on its own copy byte[] serialize = TupleFilterSerializer.serialize(originalfilter, StringCodeSystem.INSTANCE); TupleFilter filter = TupleFilterSerializer.deserialize(serialize, StringCodeSystem.INSTANCE); - + // translate FunctionTupleFilter to IN clause ITupleFilterTransformer translator = new BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap()); filter = translator.transform(filter);
