KYLIN-1851 Link into DictionaryGenerator, half way
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3fedee9f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3fedee9f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3fedee9f Branch: refs/heads/KYLIN-1851 Commit: 3fedee9fe76c979572594b85fbc6a660eb61f963 Parents: 0359fb3 Author: Li Yang <liy...@apache.org> Authored: Mon Oct 31 18:21:36 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Mon Oct 31 18:21:43 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/dict/DictionaryGenerator.java | 4 ++-- .../apache/kylin/dict/TrieDictionaryForestBuilder.java | 4 ++++ .../java/org/apache/kylin/dict/lookup/SnapshotTable.java | 11 ++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3fedee9f/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java index 0adf40e..f2c4507 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java @@ -137,7 +137,7 @@ public class DictionaryGenerator { private static class StringDictBuilder implements IDictionaryBuilder { @Override public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException { - TrieDictionaryBuilder builder = new TrieDictionaryBuilder(new StringBytesConverter()); + TrieDictionaryForestBuilder builder = new TrieDictionaryForestBuilder(new StringBytesConverter(), baseId); byte[] value; while (valueEnumerator.moveNext()) { value = valueEnumerator.current(); @@ -148,7 +148,7 @@ public class DictionaryGenerator { if (returnSamples.size() < nSamples && returnSamples.contains(v) == false) returnSamples.add(v); } - return builder.build(baseId); + return builder.build(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/3fedee9f/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java index 494ee71..4cc74ce 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java @@ -49,6 +49,10 @@ public class TrieDictionaryForestBuilder<T> { private int curOffset; + public TrieDictionaryForestBuilder(BytesConverter<T> bytesConverter) { + this(bytesConverter, 0); + } + public TrieDictionaryForestBuilder(BytesConverter<T> bytesConverter, int baseId) { this.bytesConverter = bytesConverter; this.trieBuilder = new TrieDictionaryBuilder<T>(bytesConverter); http://git-wip-us.apache.org/repos/asf/kylin/blob/3fedee9f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java index db1a170..a60b70d 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import com.google.common.base.Strings; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.kylin.common.persistence.ResourceStore; @@ -36,6 +35,7 @@ import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.dict.StringBytesConverter; import org.apache.kylin.dict.TrieDictionary; import org.apache.kylin.dict.TrieDictionaryBuilder; +import org.apache.kylin.dict.TrieDictionaryForestBuilder; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.source.ReadableTable; @@ -43,6 +43,7 @@ import org.apache.kylin.source.ReadableTable; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Strings; /** * @author yangli9 @@ -76,7 +77,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable int maxIndex = tableDesc.getMaxColumnIndex(); - TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); + TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter()); TableReader reader = table.getReader(); try { @@ -95,7 +96,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable IOUtils.closeQuietly(reader); } - this.dict = b.build(0); + this.dict = b.build(); ArrayList<int[]> allRowIndices = new ArrayList<int[]>(); reader = table.getReader(); @@ -253,7 +254,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable } } else { List<String[]> rows = new ArrayList<String[]>(rowNum); - TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); + TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter()); for (int i = 0; i < rowNum; i++) { String[] row = new String[n]; @@ -267,7 +268,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable b.addValue(row[j]); } } - this.dict = b.build(0); + this.dict = b.build(); for (String[] row : rows) { int[] rowIndex = new int[n]; for (int i = 0; i < n; i++) {
