KYLIN-2322 make TrieDictionaryForest configurable, off by default
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a78b6509 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a78b6509 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a78b6509 Branch: refs/heads/master-cdh5.7 Commit: a78b6509f8727a53e26c57b2e5a14adc0ff88b73 Parents: 4f466a0 Author: Yang Li <liy...@apache.org> Authored: Mon Dec 26 21:12:20 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Mon Dec 26 21:14:01 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 4 ++ .../apache/kylin/dict/DictionaryGenerator.java | 64 ++++++++++++++++++-- 2 files changed, 63 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a78b6509/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index da93388..8080577 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -216,6 +216,10 @@ abstract public class KylinConfigBase implements Serializable { // DICTIONARY & SNAPSHOT // ============================================================================ + public boolean isUseForestTrieDictionary() { + return Boolean.parseBoolean(getOptional("kylin.dictionary.use-forest-trie", "false")); + } + public int getTrieDictionaryForestMaxTrieSizeMB() { return Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/a78b6509/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java index cd13d59..6f4f2c4 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.metadata.datatype.DataType; @@ -49,10 +50,12 @@ public class DictionaryGenerator { builder = new DateDictBuilder(); else builder = new TimeDictBuilder(); - } else if (dataType.isNumberFamily()) { - builder = new NumberDictBuilder(); } else { - builder = new StringDictBuilder(); + boolean useForest = KylinConfig.getInstanceFromEnv().isUseForestTrieDictionary(); + if (dataType.isNumberFamily()) + builder = useForest ? new NumberTrieDictForestBuilder() : new NumberTrieDictBuilder(); + else + builder = useForest ? new StringTrieDictForestBuilder() : new StringTrieDictBuilder(); } return builder; } @@ -168,7 +171,32 @@ public class DictionaryGenerator { } } - private static class StringDictBuilder implements IDictionaryBuilder { + private static class StringTrieDictBuilder implements IDictionaryBuilder { + int baseId; + TrieDictionaryBuilder builder; + + @Override + public void init(DictionaryInfo info, int baseId) throws IOException { + this.baseId = baseId; + this.builder = new TrieDictionaryBuilder(new StringBytesConverter()); + } + + @Override + public boolean addValue(String value) { + if (value == null) + return false; + + builder.addValue(value); + return true; + } + + @Override + public Dictionary<String> build() throws IOException { + return builder.build(baseId); + } + } + + private static class StringTrieDictForestBuilder implements IDictionaryBuilder { TrieDictionaryForestBuilder builder; @Override @@ -191,7 +219,33 @@ public class DictionaryGenerator { } } - private static class NumberDictBuilder implements IDictionaryBuilder { + @SuppressWarnings("deprecation") + private static class NumberTrieDictBuilder implements IDictionaryBuilder { + int baseId; + NumberDictionaryBuilder builder; + + @Override + public void init(DictionaryInfo info, int baseId) throws IOException { + this.baseId = baseId; + this.builder = new NumberDictionaryBuilder(new StringBytesConverter()); + } + + @Override + public boolean addValue(String value) { + if (StringUtils.isBlank(value)) // empty string is treated as null + return false; + + builder.addValue(value); + return true; + } + + @Override + public Dictionary<String> build() throws IOException { + return builder.build(baseId); + } + } + + private static class NumberTrieDictForestBuilder implements IDictionaryBuilder { NumberDictionaryForestBuilder builder; @Override
