KYLIN-2154 source table rows will be skipped if TOPN's group column contains NULL values
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da2d4d0b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da2d4d0b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da2d4d0b Branch: refs/heads/yang21 Commit: da2d4d0b38c8dc1a32c4ee2049954b6a5a684ff3 Parents: f3ac9ff Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Nov 2 22:28:35 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Nov 3 10:33:57 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 9 +++++++-- .../apache/kylin/storage/gtrecord/CubeSegmentScanner.java | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/da2d4d0b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java index 3974a4f..1e2d6dd 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java @@ -18,6 +18,7 @@ package org.apache.kylin.measure.topn; +import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -142,8 +143,12 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { final ByteArray key = new ByteArray(keyLength); int offset = 0; for (int i = 0; i < dimensionEncodings.length; i++) { - byte[] valueBytes = Bytes.toBytes(values[i + 1]); - dimensionEncodings[i].encode(valueBytes, valueBytes.length, key.array(), offset); + if (values[i + 1] == null) { + Arrays.fill(key.array(), offset, offset + dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL); + } else { + byte[] valueBytes = Bytes.toBytes(values[i + 1]); + dimensionEncodings[i].encode(valueBytes, valueBytes.length, key.array(), offset); + } offset += dimensionEncodings[i].getLengthOfEncoding(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/da2d4d0b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java index 6e19c09..c6a6daa 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java @@ -53,6 +53,9 @@ public class CubeSegmentScanner implements IGTScanner { public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, // Collection<FunctionDesc> metrics, TupleFilter originalfilter, StorageContext context, String gtStorage) { + + logger.info("Init CubeSegmentScanner for segment {}", cubeSeg.getName()); + this.cuboid = cuboid; this.cubeSeg = cubeSeg; @@ -61,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner { //is working on its own copy byte[] serialize = TupleFilterSerializer.serialize(originalfilter, StringCodeSystem.INSTANCE); TupleFilter filter = TupleFilterSerializer.deserialize(serialize, StringCodeSystem.INSTANCE); - + // translate FunctionTupleFilter to IN clause ITupleFilterTransformer translator = new BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap()); filter = translator.transform(filter);
