KYLIN-2266 Reduce memory usage for building global dict Signed-off-by: shaofengshi <shaofeng...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/65c58990 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/65c58990 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/65c58990 Branch: refs/heads/master-cdh5.7 Commit: 65c58990802c644ea35005c301ba2951a9b7e6a5 Parents: 5d6bfba Author: kangkaisen <kangkai...@live.com> Authored: Sun Dec 11 20:49:28 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Mon Dec 12 10:55:32 2016 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/common/KylinConfigBase.java | 9 --------- .../java/org/apache/kylin/dict/AppendTrieDictionary.java | 6 ++---- .../org/apache/kylin/dict/AppendTrieDictionaryTest.java | 1 - 3 files changed, 2 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index f46c185..d4272f9 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -245,15 +245,6 @@ abstract public class KylinConfigBase implements Serializable { setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize)); } - public int getAppendDictCacheSize() { - return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20")); - } - - // for test - public void setAppendDictCacheSize(int cacheSize) { - setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize)); - } - public int getCachedSnapshotMaxEntrySize() { return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 35aa9a5..32bfde6 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -115,10 +115,9 @@ public class AppendTrieDictionary<T> extends Dictionary<T> { } public void initDictSliceMap(CachedTreeMap dictMap) throws IOException { - int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize(); int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions(); long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL(); - CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir) + CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir) .immutable(true).maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build(); newDictSliceMap.loadEntry(dictMap); this.dictSliceMap = newDictSliceMap; @@ -930,11 +929,10 @@ public class AppendTrieDictionary<T> extends Dictionary<T> { this.bytesConverter = bytesConverter; MAX_ENTRY_IN_SLICE = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize(); - int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize(); int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions(); long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL(); // create a new cached map with baseDir - mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir) + mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir) .maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictNode.class).immutable(false).build(); if (dictMapBytes != null) { ((Writable) mutableDictSliceMap).readFields(new DataInputStream(new ByteArrayInputStream(dictMapBytes))); http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java index a7e8152..0776599 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java @@ -63,7 +63,6 @@ public class AppendTrieDictionaryTest { System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/localmeta"); KylinConfig config = KylinConfig.getInstanceFromEnv(); config.setAppendDictEntrySize(50000); - config.setAppendDictCacheSize(3); config.setProperty("kylin.env.hdfs-working-dir", BASE_DIR); }
