Repository: kylin Updated Branches: refs/heads/master e20e2b2e8 -> 576d2dd35
KYLIN-2400 remove byte[] interface in dictionary Signed-off-by: Li Yang <liy...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e769bfa0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e769bfa0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e769bfa0 Branch: refs/heads/master Commit: e769bfa004e9d548c5a042f02765cc093ce70ec1 Parents: e20e2b2 Author: xiefan46 <958034...@qq.com> Authored: Mon Jan 9 19:11:39 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Tue Jan 17 10:54:22 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/Dictionary.java | 46 ++++---------------- .../apache/kylin/dict/AppendTrieDictionary.java | 19 ++++---- .../org/apache/kylin/dict/CacheDictionary.java | 13 ++++-- .../apache/kylin/dict/DateStrDictionary.java | 33 -------------- .../apache/kylin/dict/DictionaryGenerator.java | 2 +- .../org/apache/kylin/dict/NumberDictionary.java | 30 +------------ .../kylin/dict/NumberDictionaryBuilder.java | 8 ---- .../apache/kylin/dict/TimeStrDictionary.java | 29 ------------ .../org/apache/kylin/dict/TrieDictionary.java | 7 ++- .../kylin/dict/TrieDictionaryBuilder.java | 2 +- .../apache/kylin/dict/TrieDictionaryForest.java | 27 +++--------- .../kylin/dict/TrieDictionaryForestBuilder.java | 2 +- .../kylin/dict/AppendTrieDictionaryTest.java | 10 ++--- .../kylin/dict/DateStrDictionaryTest.java | 3 -- .../MultipleDictionaryValueEnumeratorTest.java | 14 ------ .../apache/kylin/dict/NumberDictionaryTest.java | 4 +- .../kylin/dict/TimeStrDictionaryTest.java | 9 +++- .../dict/TrieDictionaryForestBenchmark.java | 14 ++---- .../kylin/dict/TrieDictionaryForestTest.java | 11 ++--- .../apache/kylin/dict/TrieDictionaryTest.java | 19 ++++---- .../kylin/measure/bitmap/BitmapMeasureType.java | 11 +++-- .../kylin/measure/raw/RawMeasureType.java | 7 ++- .../storage/gtrecord/DictGridTableTest.java | 3 +- .../engine/mr/steps/MergeCuboidMapper.java | 11 ++--- .../mr/steps/NumberDictionaryForestTest.java | 17 +++----- 25 files changed, 92 insertions(+), 259 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java index 1e172bc..9d675f9 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java @@ -94,8 +94,13 @@ abstract public class Dictionary<T> implements Serializable { final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException { if (isNullObjectForm(value)) return nullId(); - else - return getIdFromValueImpl(value, roundingFlag); + else { + int id = getIdFromValueImpl(value, roundingFlag); + if(id == -1){ + throw new IllegalArgumentException("Value : " + value + "no exists"); + } + return id; + } } final public boolean containsValue(T value) throws IllegalArgumentException { @@ -136,9 +141,7 @@ abstract public class Dictionary<T> implements Serializable { * Convenient form of * <code>getIdFromValueBytes(value, offset, len, 0)</code> */ - final public int getIdFromValueBytes(byte[] value, int offset, int len) throws IllegalArgumentException { - return getIdFromValueBytes(value, offset, len, 0); - } + /** * A lower level API, return ID integer from raw value bytes. In case of not found @@ -153,31 +156,6 @@ abstract public class Dictionary<T> implements Serializable { * if value is not found in dictionary and rounding is off; * or if rounding cannot find a smaller or bigger ID */ - final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException { - if (isNullByteForm(value, offset, len)) - return nullId(); - else { - int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag); - if (id == -1) - throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!"); - return id; - } - } - - protected boolean isNullByteForm(byte[] value, int offset, int len) { - return value == null; - } - - abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag); - - final public byte[] getValueBytesFromId(int id) { - if (isNullId(id)) - return BytesUtil.EMPTY_BYTE_ARRAY; - else - return getValueBytesFromIdImpl(id); - } - - abstract protected byte[] getValueBytesFromIdImpl(int id); /** * A lower level API, get byte values from ID, return the number of bytes @@ -189,14 +167,6 @@ abstract public class Dictionary<T> implements Serializable { * @throws IllegalArgumentException * if ID is not found in dictionary */ - final public int getValueBytesFromId(int id, byte[] returnValue, int offset) throws IllegalArgumentException { - if (isNullId(id)) - return -1; - else - return getValueBytesFromIdImpl(id, returnValue, offset); - } - - abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset); abstract public void dump(PrintStream out); http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index b797167..3aef967 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -97,6 +97,8 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { enableCache(); } + + public void initParams(String baseDir, int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter) throws IOException { this.baseDir = baseDir; this.baseId = baseId; @@ -1113,7 +1115,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { } @Override - protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) { + protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) { if (dictSliceMap.isEmpty()) { return -1; } @@ -1129,6 +1131,11 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { } @Override + protected byte[] getValueBytesFromIdWithoutCache(int id) { + throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id"); + } + + @Override public int getMinId() { return baseId; } @@ -1149,16 +1156,6 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { } - @Override - protected byte[] getValueBytesFromIdImpl(int id) { - throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id"); - } - - @Override - protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) { - throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id"); - } - public void flushIndex(CachedTreeMap dictSliceMap, boolean keepAppend) throws IOException { try (FSDataOutputStream indexOut = dictSliceMap.openIndexOutput()) { indexOut.writeInt(baseId); http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java index b2bad53..156971d 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java @@ -52,13 +52,13 @@ public abstract class CacheDictionary<T> extends Dictionary<T> { if (id != null) return id.intValue(); byte[] valueBytes = bytesConvert.convertToBytes(value); - id = getIdFromValueBytes(valueBytes, 0, valueBytes.length, roundingFlag); + id = getIdFromValueBytesWithoutCache(valueBytes, 0, valueBytes.length, roundingFlag); cache.put(value, id); return id; } } byte[] valueBytes = bytesConvert.convertToBytes(value); - return getIdFromValueBytes(valueBytes, 0, valueBytes.length, roundingFlag); + return getIdFromValueBytesWithoutCache(valueBytes, 0, valueBytes.length, roundingFlag); } //id --> value @@ -70,13 +70,13 @@ public abstract class CacheDictionary<T> extends Dictionary<T> { int seq = calcSeqNoFromId(id); if (cache[seq] != null) return (T) cache[seq]; - byte[] valueBytes = getValueBytesFromIdImpl(id); + byte[] valueBytes = getValueBytesFromIdWithoutCache(id); T value = bytesConvert.convertFromBytes(valueBytes, 0, valueBytes.length); cache[seq] = value; return value; } } - byte[] valueBytes = getValueBytesFromIdImpl(id); + byte[] valueBytes = getValueBytesFromIdWithoutCache(id); return bytesConvert.convertFromBytes(valueBytes, 0, valueBytes.length); } @@ -99,4 +99,9 @@ public abstract class CacheDictionary<T> extends Dictionary<T> { this.valueToIdCache = null; this.idToValueCache = null; } + + abstract protected byte[] getValueBytesFromIdWithoutCache(int id); + + abstract protected int getIdFromValueBytesWithoutCache(byte[] valueBytes, int offset, int length, int roundingFlag); + } http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java index 29bbee2..56b4994 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java @@ -26,7 +26,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.io.PrintStream; -import java.io.UnsupportedEncodingException; import java.util.Date; import org.apache.commons.lang.StringUtils; @@ -84,10 +83,6 @@ public class DateStrDictionary extends Dictionary<String> { return pattern.length(); } - @Override - protected boolean isNullByteForm(byte[] value, int offset, int len) { - return value == null || len == 0; - } @Override final protected int getIdFromValueImpl(String value, int roundFlag) { @@ -107,34 +102,6 @@ public class DateStrDictionary extends Dictionary<String> { return dateToString(new Date(millis), pattern); } - @Override - final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) { - try { - return getIdFromValue(new String(value, offset, len, "ISO-8859-1")); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // never happen - } - } - - @Override - final protected byte[] getValueBytesFromIdImpl(int id) { - String date = getValueFromId(id); - byte[] bytes; - try { - bytes = date.getBytes("ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // never happen - } - return bytes; - } - - @Override - final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) { - byte[] bytes = getValueBytesFromIdImpl(id); - System.arraycopy(bytes, 0, returnValue, offset, bytes.length); - return bytes.length; - } - private int calcIdFromSeqNo(int seq) { return seq < 0 ? seq : baseId + seq; } http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java index 6f4f2c4..c23e7b5 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java @@ -227,7 +227,7 @@ public class DictionaryGenerator { @Override public void init(DictionaryInfo info, int baseId) throws IOException { this.baseId = baseId; - this.builder = new NumberDictionaryBuilder(new StringBytesConverter()); + this.builder = new NumberDictionaryBuilder(new NumberDictionaryForestBuilder.Number2BytesConverter()); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java index 9458e9d..c55937d 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java @@ -175,37 +175,9 @@ public class NumberDictionary<T> extends TrieDictionary<T> { } @Override - protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) { - NumberBytesCodec codec = getCodec(); - codec.encodeNumber(value, offset, len); - return super.getIdFromValueBytesImpl(codec.buf, codec.bufOffset, codec.bufLen, roundingFlag); - } - - @Override protected boolean isNullObjectForm(T value) { return value == null || value.equals(""); } + - @Override - protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) { - NumberBytesCodec codec = getCodec(); - codec.bufOffset = 0; - codec.bufLen = super.getValueBytesFromIdImpl(id, codec.buf, 0); - return codec.decodeNumber(returnValue, offset); - } - - - public static void main(String[] args) throws Exception { - NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<String>(new StringBytesConverter()); - b.addValue("10"); - b.addValue("100"); - b.addValue("40"); - b.addValue("7"); - TrieDictionary<String> dict = b.build(0); - - //dict.enableIdToValueBytesCache(); - for (int i = 0; i <= dict.getMaxId(); i++) { - System.out.println(Bytes.toString(dict.getValueBytesFromId(i))); - } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java index 68a05d4..b9e94414 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java @@ -18,7 +18,6 @@ package org.apache.kylin.dict; -import org.apache.kylin.common.util.Bytes; /** * Use <code>NumberDictionaryForestBuilder</code> instead. @@ -28,18 +27,11 @@ import org.apache.kylin.common.util.Bytes; @Deprecated public class NumberDictionaryBuilder<T> extends TrieDictionaryBuilder<T> { - NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec(NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT); public NumberDictionaryBuilder(BytesConverter<T> bytesConverter) { super(bytesConverter); } - @Override - void addValue(byte[] value) { - codec.encodeNumber(value, 0, value.length); - byte[] copy = Bytes.copy(codec.buf, codec.bufOffset, codec.bufLen); - super.addValue(copy); - } public NumberDictionary<T> build(int baseId) { byte[] trieBytes = buildTrieBytes(baseId); http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java index eabc9f1..d075ce1 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java @@ -22,7 +22,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.io.PrintStream; -import java.io.UnsupportedEncodingException; import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.common.util.Dictionary; @@ -85,34 +84,6 @@ public class TimeStrDictionary extends Dictionary<String> { } @Override - final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) { - try { - return getIdFromValue(new String(value, offset, len, "ISO-8859-1")); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // never happen - } - } - - @Override - final protected byte[] getValueBytesFromIdImpl(int id) { - String date = getValueFromId(id); - byte[] bytes; - try { - bytes = date.getBytes("ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // never happen - } - return bytes; - } - - @Override - final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) { - byte[] bytes = getValueBytesFromIdImpl(id); - System.arraycopy(bytes, 0, returnValue, offset, bytes.length); - return bytes.length; - } - - @Override public void dump(PrintStream out) { out.println(this.toString()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java index 8e7f5dc..9b84734 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java @@ -141,7 +141,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> { @Override - protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) { + protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) { int seq = lookupSeqNoFromValue(headSize, value, offset, offset + len, roundingFlag); int id = calcIdFromSeqNo(seq); if (id < 0) @@ -236,7 +236,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> { @Override - protected byte[] getValueBytesFromIdImpl(int id) { + protected byte[] getValueBytesFromIdWithoutCache(int id) { byte[] buf = new byte[maxValueLength]; int len = getValueBytesFromIdImpl(id, buf, 0); @@ -249,7 +249,6 @@ public class TrieDictionary<T> extends CacheDictionary<T> { } } - @Override protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) { int seq = calcSeqNoFromId(id); return lookupValueFromSeqNo(headSize, seq, returnValue, offset); @@ -425,7 +424,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> { //dict2.enableIdToValueBytesCache(); for (int i = 0; i <= dict.getMaxId(); i++) { - System.out.println(Bytes.toString(dict.getValueBytesFromId(i))); + System.out.println(Bytes.toString(dict.getValueBytesFromIdWithoutCache(i))); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java index 598865b..102c49e 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java @@ -74,7 +74,7 @@ public class TrieDictionaryBuilder<T> { // ============================================================================ private Node root; - private BytesConverter<T> bytesConverter; + protected BytesConverter<T> bytesConverter; public TrieDictionaryBuilder(BytesConverter<T> bytesConverter) { this.root = new Node(new byte[0], false); http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java index 69b4b9c..d1b5a99 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java @@ -104,15 +104,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> { } @Override - protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException { - - int result = _getIdFromValueBytesImpl(value, offset, len, roundingFlag); - //logger.info("{} => {}, rounding {}", bytesConvert.convertFromBytes(value, offset, len), result, roundingFlag); - return result; - } - - // id = tree_inner_offset + accumulate_offset + baseId - protected int _getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException { + protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException { int index; if (trees.size() == 1) { index = 0; @@ -136,27 +128,20 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> { } } TrieDictionary<T> tree = trees.get(index); - int id = tree.getIdFromValueBytes(value, offset, len, roundingFlag); + int id = tree.getIdFromValueBytesWithoutCache(value, offset, len, roundingFlag); + if(id == -1) + throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!"); id = id + accuOffset.get(index); id += baseId; return id; } @Override - protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) throws IllegalArgumentException { - int index = (trees.size() == 1) ? 0 : findIndexById(id); - int treeInnerOffset = getTreeInnerOffset(id, index); - TrieDictionary<T> tree = trees.get(index); - int size = tree.getValueBytesFromIdImpl(treeInnerOffset, returnValue, offset); - return size; - } - - @Override - protected byte[] getValueBytesFromIdImpl(int id) throws IllegalArgumentException { + protected byte[] getValueBytesFromIdWithoutCache(int id) throws IllegalArgumentException { int index = (trees.size() == 1) ? 0 : findIndexById(id); int treeInnerOffset = getTreeInnerOffset(id, index); TrieDictionary<T> tree = trees.get(index); - byte[] result = tree.getValueBytesFromId(treeInnerOffset); + byte[] result = tree.getValueBytesFromIdWithoutCache(treeInnerOffset); return result; } http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java index af2e302..69da472 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java @@ -131,7 +131,7 @@ public class TrieDictionaryForestBuilder<T> { trees.add(tree); int minId = tree.getMinId(); accuOffset.add(curOffset); - byte[] valueBytes = tree.getValueBytesFromId(minId); + byte[] valueBytes = tree.getValueBytesFromIdWithoutCache(minId); valueDivide.add(new ByteArray(valueBytes, 0, valueBytes.length)); curOffset += (tree.getMaxId() + 1); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java index 921925c..18913d0 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java @@ -180,7 +180,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { for (; checkIndex < firstAppend; checkIndex++) { String str = strList.get(checkIndex); byte[] bytes = converter.convertToBytes(str); - int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0); + int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0); assertNotEquals(String.format("Value %s not exist", str), -1, id); assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id))); checkMap.put(id, str); @@ -200,7 +200,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { for (; checkIndex < secondAppend; checkIndex++) { String str = strList.get(checkIndex); byte[] bytes = converter.convertToBytes(str); - int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0); + int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0); assertNotEquals(String.format("Value %s not exist", str), -1, id); if (checkIndex < firstAppend) { assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id)); @@ -224,7 +224,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { for (; checkIndex < strList.size(); checkIndex++) { String str = strList.get(checkIndex); byte[] bytes = converter.convertToBytes(str); - int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0); + int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0); assertNotEquals(String.format("Value %s not exist", str), -1, id); if (checkIndex < secondAppend) { assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id)); @@ -237,7 +237,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { if (notfound != null) { for (String s : notfound) { byte[] bytes = converter.convertToBytes(s); - int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0); + int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0); assertEquals(-1, id); } } @@ -245,7 +245,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { dict = testSerialize(dict, converter); for (String str : strList) { byte[] bytes = converter.convertToBytes(str); - int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0); + int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0); assertNotEquals(String.format("Value %s not exist", str), -1, id); assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java index ffc8ceb..c6d2e05 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java @@ -66,9 +66,6 @@ public class DateStrDictionaryTest { public void testNull() { int nullId = dict.getIdFromValue(null); assertNull(dict.getValueFromId(nullId)); - int nullId2 = dict.getIdFromValueBytes(null, 0, 0); - assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1); - assertEquals(nullId, nullId2); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java index 73e0935..2e90bcf 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java @@ -134,20 +134,6 @@ public class MultipleDictionaryValueEnumeratorTest { return "" + values[id]; } - @Override - protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) { - return 0; - } - - @Override - protected byte[] getValueBytesFromIdImpl(int id) { - return null; - } - - @Override - protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) { - return 0; - } @Override public void dump(PrintStream out) {} http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java index 23e925c..38f2648 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java @@ -59,7 +59,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase { @Test public void testMinMax() { - NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter()); + NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new NumberDictionaryForestBuilder.Number2BytesConverter()); builder.addValue("" + Long.MAX_VALUE); builder.addValue("" + Long.MIN_VALUE); NumberDictionary<String> dict = builder.build(0); @@ -123,7 +123,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase { int n = 100; Set<BigDecimal> set = Sets.newHashSet(); - NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter()); + NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new NumberDictionaryForestBuilder.Number2BytesConverter()); for (int i = 0; i < n; i++) { String num = randNumber(); if (set.add(new BigDecimal(num))) { http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java index ce07a86..f981942 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java @@ -23,6 +23,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.fail; + /** */ public class TimeStrDictionaryTest { @@ -62,7 +64,12 @@ public class TimeStrDictionaryTest { @Test public void testIllegal() { - Assert.assertEquals(-1, dict.getIdFromValue("2038-01-19 03:14:07")); + try{ + dict.getIdFromValue("2038-01-19 03:14:07"); + fail("should throw exception"); + }catch (IllegalArgumentException e){ + //correct + } } public void encodeDecode(String origin) { http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java index 0b4c0e3..adc1074 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java @@ -25,7 +25,6 @@ import org.junit.Test; import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Random; import java.util.UUID; @@ -123,7 +122,8 @@ public class TrieDictionaryForestBenchmark { int step = 1; for (int i = 0; i < testTimes; i++) { for (int j = 0; j < cardnality; j++) { - step |= dict.getValueBytesFromId(j).length; + //step |= dict.getValueBytesFromId(j).length; + step |= dict.getValueFromId(j).length(); } } return System.currentTimeMillis() - startTime; @@ -135,8 +135,7 @@ public class TrieDictionaryForestBenchmark { byte[] returnValue = new byte[2048]; for (int i = 0; i < testTimes; i++) { for (int j = 0; j < cardnality; j++) { - int size = dict.getValueBytesFromId(j, returnValue, 0); - step |= size; + step |= dict.getValueFromId(j).length(); } } return System.currentTimeMillis() - startTime; @@ -154,16 +153,11 @@ public class TrieDictionaryForestBenchmark { } private long runQueryIdByValueBytes(ArrayList<String> rawData, Dictionary<String> dict, int cardnality, int testTimes) { - List<byte[]> testBytes = new ArrayList<>(); - StringBytesConverter converter = new StringBytesConverter(); - for (int i = 0; i < cardnality; i++) { - testBytes.add(converter.convertToBytes(rawData.get(i))); - } long startTime = System.currentTimeMillis(); int step = 1; for (int i = 0; i < testTimes; i++) { for (int j = 0; j < cardnality; j++) { - step |= dict.getIdFromValueBytes(testBytes.get(j), 0, testBytes.get(j).length); + step |= dict.getIdFromValue(rawData.get(j)); } } return System.currentTimeMillis() - startTime; http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java index 68cf301..c7fb9c4 100755 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java @@ -641,7 +641,7 @@ public class TrieDictionaryForestTest { System.out.println("max memory:" + Runtime.getRuntime().maxMemory()); System.gc(); Thread.currentThread().sleep(1000); - NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<>(new StringBytesConverter()); + NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter()); int k = 0; while (true) { b.addValue(k + ""); @@ -820,7 +820,7 @@ public class TrieDictionaryForestTest { for (int i = 0; i < times; i++) { for (int j = 0; j < n; j++) { //System.out.println("looking for value:"+new String(array[j])); - keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length); + keep |= dict.getIdFromValueBytesWithoutCache(array[j], 0, array[j].length, 0); } } long timeValueToIdByDict = System.currentTimeMillis() - start; @@ -845,7 +845,7 @@ public class TrieDictionaryForestTest { start = System.currentTimeMillis(); for (int i = 0; i < times; i++) { for (int j = 0; j < n; j++) { - keep |= dict.getValueBytesFromId(j, valueBytes, 0); + keep |= dict.getValueBytesFromIdWithoutCache(j).length; } } long timeIdToValueByDict = System.currentTimeMillis() - start; @@ -884,7 +884,7 @@ public class TrieDictionaryForestTest { for (String s : notFound) { try { int nullId = dict.getIdFromValue(s); - //System.out.println("null value id:" + nullId); + System.out.println("null value id:" + nullId); fail("For not found value '" + s + "', IllegalArgumentException is expected"); } catch (IllegalArgumentException e) { // good @@ -905,9 +905,6 @@ public class TrieDictionaryForestTest { // test null value int nullId = dict.getIdFromValue(null); assertNull(dict.getValueFromId(nullId)); - int nullId2 = dict.getIdFromValueBytes(null, 0, 0); - assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1); - assertEquals(nullId, nullId2); } private Map<String, Integer> rightIdMap(int baseId, ArrayList<String> strs) { http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java index a87d7cb..22a93a0 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java @@ -47,7 +47,7 @@ public class TrieDictionaryTest { int count = (int) (Integer.MAX_VALUE * 0.8 / 64); benchmarkStringDictionary(new RandomStrings(count)); } - + private static class RandomStrings implements Iterable<String> { final private int size; @@ -55,7 +55,7 @@ public class TrieDictionaryTest { this.size = size; System.out.println("size = " + size); } - + @Override public Iterator<String> iterator() { return new Iterator<String>() { @@ -71,11 +71,11 @@ public class TrieDictionaryTest { public String next() { if (hasNext() == false) throw new NoSuchElementException(); - + i++; if (i % 1000000 == 0) System.out.println(i); - + return nextString(); } @@ -225,7 +225,7 @@ public class TrieDictionaryTest { TrieDictionaryBuilder<String> b = newDictBuilder(str); b.stats().print(); TrieDictionary<String> dict = b.build(0); - + TreeSet<String> set = new TreeSet<String>(); for (String s : str) { set.add(s); @@ -282,7 +282,7 @@ public class TrieDictionaryTest { start = System.currentTimeMillis(); for (int i = 0; i < times; i++) { for (int j = 0; j < n; j++) { - keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length); + keep |= dict.getIdFromValueBytesWithoutCache(array[j], 0, array[j].length, 0); } } long timeValueToIdByDict = System.currentTimeMillis() - start; @@ -304,12 +304,12 @@ public class TrieDictionaryTest { start = System.currentTimeMillis(); for (int i = 0; i < times; i++) { for (int j = 0; j < n; j++) { - keep |= dict.getValueBytesFromId(j, valueBytes, 0); + keep |= dict.getValueBytesFromIdWithoutCache(j).length; } } long timeIdToValueByDict = System.currentTimeMillis() - start; System.out.println(timeIdToValueByDict); - + return keep; } @@ -350,9 +350,6 @@ public class TrieDictionaryTest { // test null value int nullId = dict.getIdFromValue(null); assertNull(dict.getValueFromId(nullId)); - int nullId2 = dict.getIdFromValueBytes(null, 0, 0); - assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1); - assertEquals(nullId, nullId2); } private static TrieDictionary<String> testSerialize(TrieDictionary<String> dict) { http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java index b6f1975..de2a34a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java @@ -18,6 +18,8 @@ package org.apache.kylin.measure.bitmap; +import static com.google.common.base.Preconditions.checkArgument; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -38,8 +40,6 @@ import org.apache.kylin.metadata.realization.SQLDigest.SQLCall; import com.google.common.collect.ImmutableMap; -import static com.google.common.base.Preconditions.checkArgument; - /** * Created by sunyerui on 15/12/10. */ @@ -133,14 +133,13 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> { Dictionary<String> mergedDict = newDicts.get(colRef); MutableBitmapCounter retValue = new MutableBitmapCounter(); - byte[] literal = new byte[sourceDict.getSizeOfValue()]; for (int id : value) { int newId; - int size = sourceDict.getValueBytesFromId(id, literal, 0); - if (size < 0) { + String v = sourceDict.getValueFromId(id); + if (v == null) { newId = mergedDict.nullId(); } else { - newId = mergedDict.getIdFromValueBytes(literal, 0, size); + newId = mergedDict.getIdFromValue(v); } retValue.add(newId); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java index a5bb06b..3b6e659 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java @@ -133,17 +133,16 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> { int valueSize = value.size(); byte[] newIdBuf = new byte[valueSize * mergedDict.getSizeOfId()]; - byte[] literal = new byte[sourceDict.getSizeOfValue()]; int bufOffset = 0; for (ByteArray c : value) { int oldId = BytesUtil.readUnsigned(c.array(), c.offset(), c.length()); int newId; - int size = sourceDict.getValueBytesFromId(oldId, literal, 0); - if (size < 0) { + String v = sourceDict.getValueFromId(oldId); + if (v == null) { newId = mergedDict.nullId(); } else { - newId = mergedDict.getIdFromValueBytes(literal, 0, size); + newId = mergedDict.getIdFromValue(v); } BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId()); c.set(newIdBuf, bufOffset, mergedDict.getSizeOfId()); http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java index fcd0182..ae186e2 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java @@ -35,6 +35,7 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.gridtable.CubeCodeSystem; import org.apache.kylin.dict.NumberDictionaryBuilder; +import org.apache.kylin.dict.NumberDictionaryForestBuilder; import org.apache.kylin.dict.StringBytesConverter; import org.apache.kylin.dict.TrieDictionaryBuilder; import org.apache.kylin.dimension.DictionaryDimEnc; @@ -601,7 +602,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { @SuppressWarnings("rawtypes") private static Dictionary newDictionaryOfInteger() { - NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter()); + NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter()); builder.addValue("10"); builder.addValue("20"); builder.addValue("30"); http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java index a79d5aa..047e2b1 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java @@ -186,8 +186,8 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> { if (this.checkNeedMerging(col)) { // if dictionary on fact table column, needs rewrite DictionaryManager dictMgr = DictionaryManager.getInstance(config); - Dictionary<?> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col)); - Dictionary<?> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col)); + Dictionary<String> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col)); + Dictionary<String> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col)); while (sourceDict.getSizeOfValue() > newKeyBodyBuf.length - bufOffset || // mergedDict.getSizeOfValue() > newKeyBodyBuf.length - bufOffset || // @@ -200,11 +200,12 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> { int idInSourceDict = BytesUtil.readUnsigned(splittedByteses[useSplit].value, 0, splittedByteses[useSplit].length); int idInMergedDict; - int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBodyBuf, bufOffset); - if (size < 0) { + //int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBodyBuf, bufOffset); + String v = sourceDict.getValueFromId(idInSourceDict); + if (v == null) { idInMergedDict = mergedDict.nullId(); } else { - idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBodyBuf, bufOffset, size); + idInMergedDict = mergedDict.getIdFromValue(v); } BytesUtil.writeUnsigned(idInMergedDict, newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId()); http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java index f6664bf..72793f5 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java @@ -38,7 +38,6 @@ import org.apache.kylin.common.util.Bytes; import org.apache.kylin.dict.NumberDictionary; import org.apache.kylin.dict.NumberDictionaryBuilder; import org.apache.kylin.dict.NumberDictionaryForestBuilder; -import org.apache.kylin.dict.StringBytesConverter; import org.apache.kylin.dict.TrieDictionaryForest; import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag; import org.junit.Ignore; @@ -121,7 +120,7 @@ public class NumberDictionaryForestTest { TrieDictionaryForest<String> dict = b.build(); dict.dump(System.out); - NumberDictionaryBuilder<String> b2 = new NumberDictionaryBuilder<>(new StringBytesConverter()); + NumberDictionaryBuilder<String> b2 = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter()); for (String str : testData) b2.addValue(str); NumberDictionary<String> dict2 = b2.build(0); @@ -148,22 +147,20 @@ public class NumberDictionaryForestTest { assertTrue(dict1.getSizeOfId() == dict2.getSizeOfId()); assertTrue(dict1.getSizeOfValue() == dict2.getSizeOfValue()); - + byte[] buf = new byte[dict1.getSizeOfValue()]; - + { - int len = dict1.getValueBytesFromId(0, buf, 0); - int newId = dict2.getIdFromValueBytes(buf, 0, len); + int newId = dict2.getIdFromValue(dict1.getValueFromId(0)); assertTrue(newId == 0); } { - int len = dict1.getValueBytesFromId(1, buf, 0); - int newId = dict2.getIdFromValueBytes(buf, 0, len); + + int newId = dict2.getIdFromValue(dict1.getValueFromId(1)); assertTrue(newId == 2); } { - int len = dict1.getValueBytesFromId(2, buf, 0); - int newId = dict2.getIdFromValueBytes(buf, 0, len); + int newId = dict2.getIdFromValue(dict1.getValueFromId(2)); assertTrue(newId == 4); } }
