Repository: kylin Updated Branches: refs/heads/2.0-rc 9f743fb78 -> b0ecfe750
KYLIN-1306 Null strings are not applied during fast cubing Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0ecfe75 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0ecfe75 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0ecfe75 Branch: refs/heads/2.0-rc Commit: b0ecfe7507bf1d9994ff9fb76d45a868aa3347cb Parents: 9f743fb Author: shaofengshi <shaofeng...@apache.org> Authored: Tue Jan 12 16:23:04 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Jan 12 16:23:21 2016 +0800 ---------------------------------------------------------------------- .../InMemCubeBuilderInputConverter.java | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b0ecfe75/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java index 69a9fc9..4d6f758 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java @@ -18,6 +18,7 @@ package org.apache.kylin.cube.inmemcubing; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.cube.kv.RowConstants; import org.apache.kylin.cube.model.CubeDesc; @@ -40,6 +41,7 @@ import java.util.Map; public class InMemCubeBuilderInputConverter { private static final LongMutable ONE = new LongMutable(1l); + public static final byte[] HIVE_NULL = Bytes.toBytes("\\N"); private final CubeDesc cubeDesc; private final CubeJoinedFlatTableDesc intermediateTableDesc; @@ -49,6 +51,7 @@ public class InMemCubeBuilderInputConverter { private final ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); private final Map<Integer, Dictionary<String>> topNLiteralColDictMap; private final GTInfo gtInfo; + protected List<byte[]> nullBytes; public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<Integer, Dictionary<String>> topNLiteralColDictMap, GTInfo gtInfo) { @@ -59,6 +62,7 @@ public class InMemCubeBuilderInputConverter { this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); this.measureCodec = new MeasureCodec(cubeDesc.getMeasures()); this.topNLiteralColDictMap = Preconditions.checkNotNull(topNLiteralColDictMap, "topNLiteralColDictMap cannot be null"); + initNullBytes(); } public final GTRecord convert(List<String> row) { @@ -82,6 +86,9 @@ public class InMemCubeBuilderInputConverter { for (int i = 0; i < keySize; i++) { key[i] = row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]); + if (key[i] != null && isNull(Bytes.toBytes((String)key[i]))) { + key[i] = null; + } } return key; @@ -143,4 +150,24 @@ public class InMemCubeBuilderInputConverter { return v == null ? null : Bytes.toBytes(v); } + private void initNullBytes() { + nullBytes = Lists.newArrayList(); + nullBytes.add(HIVE_NULL); + String[] nullStrings = cubeDesc.getNullStrings(); + if (nullStrings != null) { + for (String s : nullStrings) { + nullBytes.add(Bytes.toBytes(s)); + } + } + } + + private boolean isNull(byte[] v) { + for (byte[] nullByte : nullBytes) { + if (Bytes.equals(v, nullByte)) + return true; + } + return false; + } + + }
