Repository: kylin Updated Branches: refs/heads/liyang-dimenc [created] 650aea4e5
http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java new file mode 100644 index 0000000..40e9f6d --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dimension; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FixedLenDimEnc extends DimensionEncoding { + + private static Logger logger = LoggerFactory.getLogger(FixedLenDimEnc.class); + + // row key fixed length place holder + public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9; + + private final int fixedLen; + + transient private int avoidVerbose = 0; + + public FixedLenDimEnc(int len) { + this.fixedLen = len; + } + + @Override + public int getLengthOfEncoding() { + return fixedLen; + } + + @Override + public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) { + if (value == null) { + Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); + return; + } + + if (valueLen > fixedLen) { + if (avoidVerbose++ % 10000 == 0) { + logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueLen + ", will truncate, value string: " + Bytes.toString(value, 0, valueLen) + " times:" + avoidVerbose); + } + } + + int n = Math.min(valueLen, fixedLen); + System.arraycopy(value, 0, output, outputOffset, n); + + if (n < fixedLen) { + Arrays.fill(output, outputOffset + n, outputOffset + fixedLen, ROWKEY_PLACE_HOLDER_BYTE); + } + } + + @Override + public String decode(byte[] bytes, int offset, int len) { + if (isNull(bytes, offset, len)) { + return null; + } + + while (len > 0 && bytes[offset + len - 1] == ROWKEY_PLACE_HOLDER_BYTE) + len--; + + return Bytes.toString(bytes, offset, len); + } + + @Override + public DataTypeSerializer<Object> asDataTypeSerializer() { + return new FixedLenSerializer(); + } + + public class FixedLenSerializer extends DataTypeSerializer<Object> { + // be thread-safe and avoid repeated obj creation + private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); + + private byte[] currentBuf() { + byte[] buf = current.get(); + if (buf == null) { + buf = new byte[fixedLen]; + current.set(buf); + } + return buf; + } + + @Override + public void serialize(Object value, ByteBuffer out) { + byte[] buf = currentBuf(); + byte[] bytes = value == null ? null : Bytes.toBytes(value.toString()); + encode(bytes, bytes == null ? 0 : bytes.length, buf, 0); + out.put(buf); + } + + @Override + public Object deserialize(ByteBuffer in) { + byte[] buf = currentBuf(); + in.get(buf); + return decode(buf, 0, buf.length); + } + + @Override + public int peekLength(ByteBuffer in) { + return fixedLen; + } + + @Override + public int maxLength() { + return fixedLen; + } + + @Override + public int getStorageBytesEstimate() { + return fixedLen; + } + + @Override + public Object valueOf(String str) { + return str; + } + }; + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java index bc387fe..550fc31 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java @@ -21,7 +21,7 @@ package org.apache.kylin.measure; import java.util.Collection; import java.util.Map; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java index 4664593..98aa752 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java @@ -23,7 +23,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java index 721ba00..b7e307f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java @@ -21,7 +21,7 @@ package org.apache.kylin.measure.basic; import java.math.BigDecimal; import java.util.Map; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java index 70ca727..0ae6e51 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java @@ -20,7 +20,7 @@ package org.apache.kylin.measure.basic; import java.util.Map; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.metadata.datatype.DoubleMutable; import org.apache.kylin.metadata.model.MeasureDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java index 2547162..5a53e6e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java @@ -20,7 +20,7 @@ package org.apache.kylin.measure.basic; import java.util.Map; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.model.MeasureDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java index df8e765..6751fd3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java @@ -1,6 +1,6 @@ package org.apache.kylin.measure.bitmap; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.measure.MeasureType; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java index ef7081c..9e5511e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java @@ -24,7 +24,7 @@ import java.util.Map; import org.apache.commons.io.Charsets; import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.measure.MeasureType; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java index 9a531f7..a90846f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java @@ -20,7 +20,7 @@ package org.apache.kylin.measure.hllc; import java.util.Map; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.measure.MeasureType; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java index 0373b07..2167cc1 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java @@ -26,7 +26,7 @@ import java.util.Map; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.measure.MeasureType; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java index 0dc1afa..b4a2b13 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java @@ -23,8 +23,8 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Set; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.kv.RowKeyColumnOrder; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java index d32f171..6ee058f 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java @@ -6,9 +6,9 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Set; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.dict.StringBytesConverter; import org.apache.kylin.dict.TrieDictionaryBuilder; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TableDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java index 3dddece..bc5831e 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java @@ -10,7 +10,6 @@ import org.apache.hadoop.io.Text; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesSplitter; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.SplittedBytes; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; @@ -20,6 +19,7 @@ import org.apache.kylin.cube.kv.AbstractRowKeyEncoder; import org.apache.kylin.cube.kv.RowConstants; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.engine.mr.KylinMapper; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; import org.apache.kylin.engine.mr.common.BatchConstants; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java index b094b98..4955d58 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java @@ -16,13 +16,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.MemoryBudgetController; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.engine.mr.ByteArrayWritable; import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat; import org.apache.kylin.engine.mr.KylinMapper; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java index 682a91c..6766b31 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java @@ -31,7 +31,6 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.Pair; import org.apache.kylin.common.util.SplittedBytes; import org.apache.kylin.cube.CubeInstance; @@ -44,6 +43,7 @@ import org.apache.kylin.cube.kv.RowKeyEncoder; import org.apache.kylin.cube.kv.RowKeyEncoderProvider; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.dict.DictionaryManager; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.engine.mr.KylinMapper; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; import org.apache.kylin.engine.mr.common.BatchConstants; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java index 5db3631..5fceff7 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java @@ -29,7 +29,6 @@ import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.mrunit.mapreduce.MapDriver; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; @@ -40,6 +39,7 @@ import org.apache.kylin.dict.DictionaryInfo; import org.apache.kylin.dict.DictionaryManager; import org.apache.kylin.dict.IterableDictionaryValueEnumerator; import org.apache.kylin.dict.TrieDictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java index 285729f..6495cac 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java @@ -43,7 +43,6 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; @@ -56,6 +55,7 @@ import org.apache.kylin.cube.kv.RowConstants; import org.apache.kylin.cube.model.*; import org.apache.kylin.cube.util.CubingUtils; import org.apache.kylin.dict.*; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.engine.mr.common.CubeStatsReader; import org.apache.kylin.engine.spark.cube.BufferedCuboidWriter; import org.apache.kylin.engine.spark.cube.DefaultTupleConverter; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java ---------------------------------------------------------------------- diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java index 3fbade2..234fdf5 100644 --- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java +++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java @@ -36,8 +36,8 @@ package org.apache.kylin.engine.streaming; import java.util.Map; import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.StreamingBatch; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.engine.streaming.util.StreamingUtils; import org.apache.kylin.metadata.model.IBuildable; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java ---------------------------------------------------------------------- diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java index 7946438..41f581a 100644 --- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java +++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java @@ -36,9 +36,9 @@ package org.apache.kylin.engine.streaming; import java.util.Map; import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.StreamingBatch; import org.apache.kylin.cube.inmemcubing.ICuboidWriter; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.model.IBuildable; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java ---------------------------------------------------------------------- diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java index d7056cf..f3a2835 100644 --- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java +++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java @@ -53,10 +53,10 @@ import org.apache.kylin.cube.CubeUpdate; import org.apache.kylin.cube.inmemcubing.ICuboidWriter; import org.apache.kylin.cube.inmemcubing.InMemCubeBuilder; import org.apache.kylin.cube.util.CubingUtils; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.StreamingBatch; import org.apache.kylin.engine.streaming.StreamingBatchBuilder; import org.apache.kylin.common.util.StreamingMessage; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.model.IBuildable; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java index dedf8de..e3604c7 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java @@ -22,8 +22,8 @@ import java.text.SimpleDateFormat; import java.util.List; import java.util.TimeZone; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.dict.IDictionaryAware; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.index.TableRecordInfo; import org.apache.kylin.invertedindex.model.IIDesc; import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java index e395544..b4511e1 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java @@ -24,7 +24,7 @@ import java.util.Arrays; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import com.ning.compress.lzf.LZFDecoder; import com.ning.compress.lzf.LZFEncoder; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java index 2d635ab..d42cab0 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java @@ -23,7 +23,7 @@ import java.util.Arrays; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec; import org.apache.kylin.metadata.datatype.LongMutable; @@ -45,7 +45,7 @@ public class RawTableRecord implements Cloneable { } public void reset() { - Arrays.fill(buf, Dictionary.NULL); + Arrays.fill(buf, DimensionEncoding.NULL); } public boolean isMetric(int col) { http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java index dc2c5c4..5611f6e 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java @@ -21,7 +21,7 @@ package org.apache.kylin.invertedindex.index; import java.util.Iterator; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import com.google.common.base.Objects; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java index 792a0cc..d80370b 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java @@ -20,9 +20,9 @@ package org.apache.kylin.invertedindex.index; import com.google.common.base.Function; import com.google.common.collect.Lists; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.StreamingBatch; import org.apache.kylin.common.util.StreamingMessage; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.model.IIDesc; import org.apache.kylin.invertedindex.util.IIDictionaryBuilder; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java index 3ee34be..5ac73c7 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java @@ -23,8 +23,8 @@ import java.util.Arrays; import org.apache.commons.lang.ObjectUtils; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.ShardingHash; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.datatype.LongMutable; /** http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java index 628a08d..3778c2a 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java @@ -21,7 +21,7 @@ package org.apache.kylin.invertedindex.index; import java.util.List; import org.apache.kylin.common.util.Array; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.IISegment; import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec; import org.apache.kylin.invertedindex.model.IIDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java index 73a605c..802e629 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java @@ -26,8 +26,8 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.dict.DictionarySerializer; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.index.ColumnValueContainer; import org.apache.kylin.invertedindex.index.CompressedValueContainer; import org.apache.kylin.invertedindex.index.Slice; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java index 4187193..9177e5d 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java @@ -40,9 +40,9 @@ import java.util.List; import javax.annotation.Nullable; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.dict.DictionaryGenerator; import org.apache.kylin.dict.IterableDictionaryValueEnumerator; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.model.IIDesc; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java index 6047514..f40084d 100644 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java +++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java @@ -21,8 +21,8 @@ package org.apache.kylin.invertedindex; import java.io.IOException; import java.util.List; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.IIDescManager; import org.apache.kylin.invertedindex.IIInstance; import org.apache.kylin.invertedindex.IIManager; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java index 8bd19ab..4b74fdd 100644 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java +++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java @@ -34,10 +34,10 @@ import javax.annotation.Nullable; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.dict.DictionaryGenerator; import org.apache.kylin.dict.IterableDictionaryValueEnumerator; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.IIInstance; import org.apache.kylin.invertedindex.IIManager; import org.apache.kylin.invertedindex.index.CompressedValueContainer; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java index 3fbc69b..b8d4f55 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java @@ -4,7 +4,7 @@ import java.util.Arrays; import java.util.LinkedList; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.cube.kv.RowConstants; +import org.apache.kylin.dimension.FixedLenDimEnc; import com.google.common.collect.Lists; @@ -44,7 +44,7 @@ public class AggrKey implements Comparable<AggrKey> { int hash = 1; for (int i = 0; i < groupByMaskSet.length; i++) { byte t = data[offset + groupByMaskSet[i]]; - if (t != RowConstants.ROWKEY_PLACE_HOLDER_BYTE) { + if (t != FixedLenDimEnc.ROWKEY_PLACE_HOLDER_BYTE) { hash = (31 * hash) + t; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java index 294f399..13c3759 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java @@ -4,11 +4,12 @@ import java.util.Collection; import java.util.Set; import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.kv.RowKeyColumnIO; import org.apache.kylin.dict.DictCodeSystem; import org.apache.kylin.dict.IDictionaryAware; import org.apache.kylin.dict.TupleFilterDictionaryTranslater; +import org.apache.kylin.dimension.Dictionary; +import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; @@ -182,7 +183,7 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator { private String translate(TblColRef column, String v, int roundingFlag) { byte[] value = Bytes.toBytes(v); byte[] id = new byte[columnIO.getColumnLength(column)]; - columnIO.writeColumn(column, value, value.length, roundingFlag, Dictionary.NULL, id, 0); + columnIO.writeColumn(column, value, value.length, roundingFlag, DimensionEncoding.NULL, id, 0); return Dictionary.dictIdToString(id, 0, id.length); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 1b8b586..31c5193 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -36,7 +36,6 @@ import java.util.TreeSet; import org.apache.hadoop.hbase.client.HConnection; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.Pair; import org.apache.kylin.common.util.ShardingHash; import org.apache.kylin.cube.CubeInstance; @@ -49,6 +48,7 @@ import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.cube.model.HBaseColumnDesc; import org.apache.kylin.cube.model.HBaseMappingDesc; import org.apache.kylin.dict.lookup.LookupStringTable; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java index 4f674cb..d1a12ec 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java @@ -8,7 +8,6 @@ import java.util.Map.Entry; import org.apache.hadoop.hbase.client.Result; import org.apache.kylin.common.util.Array; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; @@ -16,6 +15,7 @@ import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.kv.RowKeyDecoder; import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.dict.lookup.LookupStringTable; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java index 06c6e2c..8cc11cf 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java @@ -19,7 +19,7 @@ package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java index a7346af..1f586b3 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java @@ -24,13 +24,13 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.kylin.common.util.Array; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.dict.lookup.LookupStringTable; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java index 1f024fe..4dee3be 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java @@ -20,7 +20,7 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint; import java.util.List; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; import org.apache.kylin.metadata.filter.LogicalTupleFilter; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java index 41a2584..2fd283c 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java @@ -36,8 +36,8 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint; import java.util.Map; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.dict.IDictionaryAware; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.index.TableRecordInfo; import org.apache.kylin.invertedindex.index.TableRecordInfoDigest; import org.apache.kylin.metadata.model.TblColRef; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java index af7b993..48802cb 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java @@ -18,6 +18,8 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint; +import it.uniroma3.mat.extendedset.intset.ConciseSet; + import java.io.IOException; import java.util.Arrays; import java.util.Iterator; @@ -40,9 +42,10 @@ import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.BytesSerializer; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.common.util.CompressionUtils; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.kv.RowKeyColumnIO; import org.apache.kylin.dict.TrieDictionary; +import org.apache.kylin.dimension.Dictionary; +import org.apache.kylin.dimension.FixedLenDimEnc; import org.apache.kylin.invertedindex.index.RawTableRecord; import org.apache.kylin.invertedindex.index.Slice; import org.apache.kylin.invertedindex.index.TableRecordInfoDigest; @@ -65,7 +68,6 @@ import com.google.protobuf.HBaseZeroCopyByteString; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; import com.google.protobuf.Service; -import it.uniroma3.mat.extendedset.intset.ConciseSet; /** */ @@ -292,19 +294,28 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop final boolean emptyDictionary = Array.isEmpty(localDictionaries); for (int i = 0; i < finalColumnLengths.length; i++) { if (isMetric[i]) { - rowKeyColumnIO.writeColumnWithoutDictionary(encodedRecord.getBytes(), encodedRecord.offset(i), encodedRecord.length(i), recordBuffer, digest.offset(i), finalColumnLengths[i]); + writeColumnWithoutDictionary(encodedRecord.getBytes(), encodedRecord.offset(i), encodedRecord.length(i), recordBuffer, digest.offset(i), finalColumnLengths[i]); } else { if (emptyDictionary) { - rowKeyColumnIO.writeColumnWithoutDictionary(encodedRecord.getBytes(), encodedRecord.offset(i), encodedRecord.length(i), recordBuffer, digest.offset(i), finalColumnLengths[i]); + writeColumnWithoutDictionary(encodedRecord.getBytes(), encodedRecord.offset(i), encodedRecord.length(i), recordBuffer, digest.offset(i), finalColumnLengths[i]); } else { final Dictionary<?> localDictionary = localDictionaries[i]; final byte[] valueBytesFromId = localDictionary.getValueBytesFromId(encodedRecord.getValueID(i)); - rowKeyColumnIO.writeColumnWithoutDictionary(valueBytesFromId, 0, valueBytesFromId.length, recordBuffer, digest.offset(i), finalColumnLengths[i]); + writeColumnWithoutDictionary(valueBytesFromId, 0, valueBytesFromId.length, recordBuffer, digest.offset(i), finalColumnLengths[i]); } } } } + private void writeColumnWithoutDictionary(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength) { + if (srcLength >= dstLength) { + System.arraycopy(src, srcOffset, dst, dstOffset, dstLength); + } else { + System.arraycopy(src, srcOffset, dst, dstOffset, srcLength); + Arrays.fill(dst, dstOffset + srcLength, dstOffset + dstLength, FixedLenDimEnc.ROWKEY_PLACE_HOLDER_BYTE); + } + } + @Override public void start(CoprocessorEnvironment env) throws IOException { if (env instanceof RegionCoprocessorEnvironment) { http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java index 94e5b33..2e31e81 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java @@ -1,7 +1,7 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint; -import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.dict.IDictionaryAware; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.invertedindex.index.TableRecordInfoDigest; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType; http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java index 61df5cf..6f82f83 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals; import java.util.ArrayList; -import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.Dictionary; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter;
