http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java index 2747ca0..f749fb4 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java @@ -93,10 +93,10 @@ public class AggregationCacheMemSizeTest { } enum Settings { - WITHOUT_MEM_HUNGRY, // only test basic aggrs - WITH_HLLC, // basic aggrs + hllc - WITH_LOW_CARD_BITMAP, // basic aggrs + bitmap - WITH_HIGH_CARD_BITMAP // basic aggrs + bitmap + WITHOUT_MEM_HUNGRY, // only test basic aggrs + WITH_HLLC, // basic aggrs + hllc + WITH_LOW_CARD_BITMAP, // basic aggrs + bitmap + WITH_HIGH_CARD_BITMAP // basic aggrs + bitmap } private MeasureAggregator<?>[] createNoMemHungryAggrs() { @@ -132,19 +132,19 @@ public class AggregationCacheMemSizeTest { aggregators.addAll(Arrays.asList(createNoMemHungryAggrs())); switch (settings) { - case WITHOUT_MEM_HUNGRY: - break; - case WITH_HLLC: - aggregators.add(createHLLCAggr()); - break; - case WITH_LOW_CARD_BITMAP: - aggregators.add(createBitmapAggr(true)); - break; - case WITH_HIGH_CARD_BITMAP: - aggregators.add(createBitmapAggr(false)); - break; - default: - break; + case WITHOUT_MEM_HUNGRY: + break; + case WITH_HLLC: + aggregators.add(createHLLCAggr()); + break; + case WITH_LOW_CARD_BITMAP: + aggregators.add(createBitmapAggr(true)); + break; + case WITH_HIGH_CARD_BITMAP: + aggregators.add(createBitmapAggr(false)); + break; + default: + break; } return aggregators.toArray(new MeasureAggregator[aggregators.size()]); @@ -160,8 +160,8 @@ public class AggregationCacheMemSizeTest { System.out.printf("%-15s %-10s %-10s\n", "cardinality", "estimate", "actual"); for (BitmapAggregator aggr : bitmapAggrs) { - System.out.printf("%-15d %-10d %-10d\n", aggr.getState().getCount(), aggr.getMemBytesEstimate(), - meter.measureDeep(aggr)); + System.out.printf("%-15d %-10d %-10d\n", + aggr.getState().getCount(), aggr.getMemBytesEstimate(), meter.measureDeep(aggr)); } } @@ -190,8 +190,8 @@ public class AggregationCacheMemSizeTest { long actualMillis = 0; System.out.println("Settings: " + settings); - System.out.printf("%15s %15s %15s %15s %15s\n", "Size", "Estimate(bytes)", "Actual(bytes)", "Estimate(ms)", - "Actual(ms)"); + System.out.printf("%15s %15s %15s %15s %15s\n", + "Size", "Estimate(bytes)", "Actual(bytes)", "Estimate(ms)", "Actual(ms)"); for (int i = 0; i < inputCount; i++) { byte[] key = new byte[10]; @@ -199,7 +199,7 @@ public class AggregationCacheMemSizeTest { MeasureAggregator[] values = createAggrs(settings); map.put(key, values); - if ((i + 1) % reportInterval == 0) { + if ((i+1) % reportInterval == 0) { stopwatch.start(); long estimateBytes = GTAggregateScanner.estimateSizeOfAggrCache(key, values, map.size()); estimateMillis += stopwatch.elapsedMillis(); @@ -210,8 +210,8 @@ public class AggregationCacheMemSizeTest { actualMillis += stopwatch.elapsedMillis(); stopwatch.reset(); - System.out.printf("%,15d %,15d %,15d %,15d %,15d\n", map.size(), estimateBytes, actualBytes, - estimateMillis, actualMillis); + System.out.printf("%,15d %,15d %,15d %,15d %,15d\n", + map.size(), estimateBytes, actualBytes, estimateMillis, actualMillis); } } System.out.println("---------------------------------------\n");
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java index d9241e5..8b2243c 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java @@ -77,11 +77,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { } }; - GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null) - .setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)) - .setAggrMetrics(new ImmutableBitSet(3, 6)) - .setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null) - .setAggCacheMemThreshold(0.5).createGTScanRequest(); + GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest(); GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest); @@ -118,11 +114,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { }; // all-in-mem testcase - GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null) - .setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3)) - .setAggrMetrics(new ImmutableBitSet(3, 6)) - .setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null) - .setAggCacheMemThreshold(0.5).createGTScanRequest(); + GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest(); GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java index 770f6ce..1866079 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java @@ -45,13 +45,9 @@ public class DimEncodingPreserveOrderTest { successValue.add(new long[] { -32767, -127, 0, 127, 32767 }); successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 8388607 }); successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L }); - successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, - 2147483647L, 549755813887L }); - successValue.add(new long[] { -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, - 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L }); - successValue.add(new long[] { -36028797018963967L, -140737488355327L, -549755813887L, -2147483647L, -8388607, - -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L, - 36028797018963967L }); + successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L }); + successValue.add(new long[] { -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L }); + successValue.add(new long[] { -36028797018963967L, -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L, 36028797018963967L }); successValue.add(new long[] { // -9223372036854775807L, // -36028797018963967L, // http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java index aadcf45..1ae229a 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java @@ -88,8 +88,7 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase { @Test public void testGTInfo() { - CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()) - .getCube("test_kylin_cube_with_slr_ready"); + CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready"); CubeSegment segment = cube.getFirstSegment(); Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java index 58bb76c..14a25c5 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java @@ -85,8 +85,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { } private IGTScanner scan(GridTable table) throws IOException { - GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null) - .setFilterPushDown(null).createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = table.scan(req); for (GTRecord r : scanner) { Object[] v = r.getValues(); @@ -101,9 +100,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { } private IGTScanner scanAndAggregate(GridTable table) throws IOException { - GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null) - .setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)) - .setAggrMetricsFuncs(new String[] { "count", "sum" }).setFilterPushDown(null).createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[] { "count", "sum" }).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java index a11b84a..7129a5e 100644 --- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java +++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java @@ -50,8 +50,7 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase { @Test public void basicTest() { - MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), - measure("HLLC16"), measure("bitmap") }; + MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") }; BufferedMeasureCodec codec = new BufferedMeasureCodec(descs); Double d = new Double(1.0); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java index bda2bf0..f04c742 100644 --- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java +++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java @@ -51,8 +51,7 @@ public class TopNMeasureTypeTest extends LocalFileMetadataTestCase { @Test public void test() { - CubeDesc desc = CubeDescManager.getInstance(getTestConfig()) - .getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); + CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); MeasureDesc topSellerMeasure = null; @@ -62,8 +61,7 @@ public class TopNMeasureTypeTest extends LocalFileMetadataTestCase { break; } } - TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create( - topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType()); + TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType()); topSellerMeasure.getFunction().getConfiguration().clear(); List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); @@ -71,8 +69,7 @@ public class TopNMeasureTypeTest extends LocalFileMetadataTestCase { assertTrue(colsNeedDict != null && colsNeedDict.size() == 1); TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1); - topSellerMeasure.getFunction().getConfiguration() - .put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6"); + topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6"); colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict.size() == 0); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 9df0e26..b9f0d2b 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -17,16 +17,11 @@ */ package org.apache.kylin.dict; -import static com.google.common.base.Preconditions.checkState; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Arrays; -import java.util.Objects; -import java.util.concurrent.ExecutionException; - +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; import org.apache.hadoop.fs.Path; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Dictionary; @@ -37,12 +32,14 @@ import org.apache.kylin.dict.global.GlobalDictMetadata; import org.apache.kylin.dict.global.GlobalDictStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import static com.google.common.base.Preconditions.checkState; /** * A dictionary based on Trie data structure that maps enumerations of byte[] to @@ -63,8 +60,7 @@ import com.google.common.cache.RemovalNotification; */ @SuppressWarnings({ "rawtypes", "unchecked", "serial" }) public class AppendTrieDictionary<T> extends CacheDictionary<T> { - public static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, - 0x44, 0x69, 0x63, 0x74 }; // "AppendTrieDict" + public static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "AppendTrieDict" public static final int HEAD_SIZE_I = HEAD_MAGIC.length; private static final Logger logger = LoggerFactory.getLogger(AppendTrieDictionary.class); @@ -81,23 +77,19 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { final Path latestVersionPath = globalDictStore.getVersionDir(latestVersion); this.metadata = globalDictStore.getMetadata(latestVersion); this.bytesConvert = metadata.bytesConverter; - this.dictCache = CacheBuilder.newBuilder().softValues() - .removalListener(new RemovalListener<AppendDictSliceKey, AppendDictSlice>() { - @Override - public void onRemoval(RemovalNotification<AppendDictSliceKey, AppendDictSlice> notification) { - logger.info("Evict slice with key {} and value {} caused by {}, size {}/{}", - notification.getKey(), notification.getValue(), notification.getCause(), - dictCache.size(), metadata.sliceFileMap.size()); - } - }).build(new CacheLoader<AppendDictSliceKey, AppendDictSlice>() { - @Override - public AppendDictSlice load(AppendDictSliceKey key) throws Exception { - AppendDictSlice slice = globalDictStore.readSlice(latestVersionPath.toString(), - metadata.sliceFileMap.get(key)); - logger.info("Load slice with key {} and value {}", key, slice); - return slice; - } - }); + this.dictCache = CacheBuilder.newBuilder().softValues().removalListener(new RemovalListener<AppendDictSliceKey, AppendDictSlice>() { + @Override + public void onRemoval(RemovalNotification<AppendDictSliceKey, AppendDictSlice> notification) { + logger.info("Evict slice with key {} and value {} caused by {}, size {}/{}", notification.getKey(), notification.getValue(), notification.getCause(), dictCache.size(), metadata.sliceFileMap.size()); + } + }).build(new CacheLoader<AppendDictSliceKey, AppendDictSlice>() { + @Override + public AppendDictSlice load(AppendDictSliceKey key) throws Exception { + AppendDictSlice slice = globalDictStore.readSlice(latestVersionPath.toString(), metadata.sliceFileMap.get(key)); + logger.info("Load slice with key {} and value {}", key, slice); + return slice; + } + }); } @Override @@ -191,4 +183,4 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { public boolean contains(Dictionary other) { return false; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java index 9eafc43..7a48a1c 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java @@ -70,8 +70,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer { } } else if (tupleFilter instanceof LogicalTupleFilter) { @SuppressWarnings("unchecked") - ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren() - .listIterator(); + ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); while (childIterator.hasNext()) { TupleFilter transformed = transform(childIterator.next()); if (transformed != null) { @@ -93,8 +92,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer { if (dict == null) return null; - CompareTupleFilter translated = new CompareTupleFilter( - builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN); + CompareTupleFilter translated = new CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN); translated.addChild(new ColumnTupleFilter(columnRef)); try { @@ -120,12 +118,10 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer { @SuppressWarnings({ "unchecked", "rawtypes" }) private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { - if (compTupleFilter.getFunction() == null - || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter))) + if (compTupleFilter.getFunction() == null || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter))) return null; - BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter - .getFunction(); + BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter.getFunction(); if (!builtInFunctionTupleFilter.isValid()) return null; @@ -135,8 +131,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer { if (dict == null) return null; - CompareTupleFilter translated = new CompareTupleFilter( - builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN); + CompareTupleFilter translated = new CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN); translated.addChild(new ColumnTupleFilter(columnRef)); try { @@ -147,8 +142,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer { Class clazz = Primitives.wrap(computedVal.getClass()); Object targetVal = compTupleFilter.getFirstValue(); if (Primitives.isWrapperType(clazz)) - targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, - compTupleFilter.getFirstValue())); + targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); int comp = ((Comparable) computedVal).compareTo(targetVal); boolean compResult = false; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java index 798c9c9..74d5ec5 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java @@ -18,10 +18,10 @@ package org.apache.kylin.dict; -import java.util.Comparator; - import org.apache.kylin.common.util.ByteArray; +import java.util.Comparator; + /** * Created by xiefan on 16-10-28. */ http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java index 15e28a6..156971d 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java @@ -18,12 +18,12 @@ package org.apache.kylin.dict; +import org.apache.kylin.common.util.Dictionary; + import java.lang.ref.SoftReference; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.apache.kylin.common.util.Dictionary; - /** */ public abstract class CacheDictionary<T> extends Dictionary<T> { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java index f772caf..56b4994 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java @@ -83,13 +83,13 @@ public class DateStrDictionary extends Dictionary<String> { return pattern.length(); } + @Override final protected int getIdFromValueImpl(String value, int roundFlag) { Date date = stringToDate(value, pattern); int id = calcIdFromSeqNo((int) DateDimEnc.getNumOfDaysSince0000FromMillis(date.getTime())); if (id < baseId || id > maxId) - throw new IllegalArgumentException( - "'" + value + "' encodes to '" + id + "' which is out of range [" + baseId + "," + maxId + "]"); + throw new IllegalArgumentException("'" + value + "' encodes to '" + id + "' which is out of range [" + baseId + "," + maxId + "]"); return id; } @@ -123,6 +123,7 @@ public class DateStrDictionary extends Dictionary<String> { init(pattern, baseId); } + @Override public int hashCode() { return 31 * baseId + pattern.hashCode(); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java index 14faff4..61a0664 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java @@ -60,13 +60,11 @@ public class DictionaryGenerator { return builder; } - public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator) - throws IOException { + public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException { return buildDictionary(newDictionaryBuilder(dataType), null, valueEnumerator); } - static Dictionary<String> buildDictionary(IDictionaryBuilder builder, DictionaryInfo dictInfo, - IDictionaryValueEnumerator valueEnumerator) throws IOException { + static Dictionary<String> buildDictionary(IDictionaryBuilder builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) throws IOException { int baseId = 0; // always 0 for now int nSamples = 5; ArrayList<String> samples = new ArrayList<String>(nSamples); @@ -121,7 +119,7 @@ public class DictionaryGenerator { public boolean addValue(String value) { if (StringUtils.isBlank(value)) // empty string is treated as null return false; - + // detect date pattern on the first value if (datePattern == null) { for (String p : DATE_PATTERNS) { @@ -136,7 +134,7 @@ public class DictionaryGenerator { if (datePattern == null) throw new IllegalArgumentException("Unknown date pattern for input value: " + value); } - + // check the date format DateFormat.stringToDate(value, datePattern); return true; @@ -176,28 +174,28 @@ public class DictionaryGenerator { private static class StringTrieDictBuilder implements IDictionaryBuilder { int baseId; TrieDictionaryBuilder builder; - + @Override public void init(DictionaryInfo info, int baseId) throws IOException { this.baseId = baseId; this.builder = new TrieDictionaryBuilder(new StringBytesConverter()); } - + @Override public boolean addValue(String value) { if (value == null) return false; - + builder.addValue(value); return true; } - + @Override public Dictionary<String> build() throws IOException { return builder.build(baseId); } } - + private static class StringTrieDictForestBuilder implements IDictionaryBuilder { TrieDictionaryForestBuilder builder; @@ -225,28 +223,28 @@ public class DictionaryGenerator { private static class NumberTrieDictBuilder implements IDictionaryBuilder { int baseId; NumberDictionaryBuilder builder; - + @Override public void init(DictionaryInfo info, int baseId) throws IOException { this.baseId = baseId; this.builder = new NumberDictionaryBuilder(); } - + @Override public boolean addValue(String value) { if (StringUtils.isBlank(value)) // empty string is treated as null return false; - + builder.addValue(value); return true; } - + @Override public Dictionary<String> build() throws IOException { return builder.build(baseId); } } - + private static class NumberTrieDictForestBuilder implements IDictionaryBuilder { NumberDictionaryForestBuilder builder; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java index 6d27d98..ae5c0f1 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java @@ -55,9 +55,8 @@ public class DictionaryInfo extends RootPersistentEntity { public DictionaryInfo(ColumnDesc col, String dataType, TableSignature input) { this(col.getTable().getIdentity(), col.getName(), col.getZeroBasedIndex(), dataType, input); } - - public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, - TableSignature input) { + + public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input) { this.updateRandomUuid(); @@ -94,10 +93,7 @@ public class DictionaryInfo extends RootPersistentEntity { // to decide if two dictionaries are built on the same table/column, // regardless of their signature public boolean isDictOnSameColumn(DictionaryInfo other) { - return this.sourceTable.equalsIgnoreCase(other.sourceTable) - && this.sourceColumn.equalsIgnoreCase(other.sourceColumn) - && this.sourceColumnIndex == other.sourceColumnIndex && this.dataType.equalsIgnoreCase(other.dataType) - && this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass); + return this.sourceTable.equalsIgnoreCase(other.sourceTable) && this.sourceColumn.equalsIgnoreCase(other.sourceColumn) && this.sourceColumnIndex == other.sourceColumnIndex && this.dataType.equalsIgnoreCase(other.dataType) && this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass); } public String getSourceTable() { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index aa84f3a..1628f4e 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -95,8 +95,7 @@ public class DictionaryManager { .removalListener(new RemovalListener<String, DictionaryInfo>() { @Override public void onRemoval(RemovalNotification<String, DictionaryInfo> notification) { - DictionaryManager.logger.info("Dict with resource path " + notification.getKey() - + " is removed due to " + notification.getCause()); + DictionaryManager.logger.info("Dict with resource path " + notification.getKey() + " is removed due to " + notification.getCause()); } })// .maximumSize(config.getCachedDictMaxEntrySize())// @@ -156,8 +155,7 @@ public class DictionaryManager { largestDictInfo = getDictionaryInfo(largestDictInfo.getResourcePath()); Dictionary<String> largestDictObject = largestDictInfo.getDictionaryObject(); if (largestDictObject.contains(newDict)) { - logger.info("dictionary content " + newDict + ", is contained by dictionary at " - + largestDictInfo.getResourcePath()); + logger.info("dictionary content " + newDict + ", is contained by dictionary at " + largestDictInfo.getResourcePath()); return largestDictInfo; } else if (newDict.contains(largestDictObject)) { logger.info("dictionary content " + newDict + " is by far the largest, save it"); @@ -233,8 +231,7 @@ public class DictionaryManager { } else { if (!firstDictInfo.isDictOnSameColumn(info)) { // don't throw exception, just output warning as legacy cube segment may build dict on PK - logger.warn("Merging dictionaries are not structurally equal : " + firstDictInfo.getResourcePath() - + " and " + info.getResourcePath()); + logger.warn("Merging dictionaries are not structurally equal : " + firstDictInfo.getResourcePath() + " and " + info.getResourcePath()); } } totalSize += info.getInput().getSize(); @@ -270,19 +267,16 @@ public class DictionaryManager { logger.info("Use one of the merging dictionaries directly"); return dicts.get(0); } else { - Dictionary<String> newDict = DictionaryGenerator - .mergeDictionaries(DataType.getType(newDictInfo.getDataType()), dicts); + Dictionary<String> newDict = DictionaryGenerator.mergeDictionaries(DataType.getType(newDictInfo.getDataType()), dicts); return trySaveNewDict(newDict, newDictInfo); } } - public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable) - throws IOException { + public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable) throws IOException { return buildDictionary(model, col, inpTable, null); } - public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable, - String builderClass) throws IOException { + public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable, String builderClass) throws IOException { if (inpTable.exists() == false) return null; @@ -291,8 +285,7 @@ public class DictionaryManager { DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable); String dupInfo = checkDupByInfo(dictInfo); if (dupInfo != null) { - logger.info( - "Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo); + logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo); return getDictionaryInfo(dupInfo); } @@ -303,16 +296,13 @@ public class DictionaryManager { return trySaveNewDict(dictionary, dictInfo); } - private Dictionary<String> buildDictFromReadableTable(IReadableTable inpTable, DictionaryInfo dictInfo, - String builderClass, TblColRef col) throws IOException { + private Dictionary<String> buildDictFromReadableTable(IReadableTable inpTable, DictionaryInfo dictInfo, String builderClass, TblColRef col) throws IOException { Dictionary<String> dictionary; IDictionaryValueEnumerator columnValueEnumerator = null; try { - columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(), - dictInfo.getSourceColumnIndex()); + columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(), dictInfo.getSourceColumnIndex()); if (builderClass == null) { - dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()), - columnValueEnumerator); + dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()), columnValueEnumerator); } else { IDictionaryBuilder builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass); dictionary = DictionaryGenerator.buildDictionary(builder, dictInfo, columnValueEnumerator); @@ -326,21 +316,18 @@ public class DictionaryManager { return dictionary; } - public DictionaryInfo saveDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable, - Dictionary<String> dictionary) throws IOException { + public DictionaryInfo saveDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable, Dictionary<String> dictionary) throws IOException { DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable); String dupInfo = checkDupByInfo(dictInfo); if (dupInfo != null) { - logger.info( - "Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo); + logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo); return getDictionaryInfo(dupInfo); } return trySaveNewDict(dictionary, dictInfo); } - private DictionaryInfo createDictionaryInfo(DataModelDesc model, TblColRef col, IReadableTable inpTable) - throws IOException { + private DictionaryInfo createDictionaryInfo(DataModelDesc model, TblColRef col, IReadableTable inpTable) throws IOException { TblColRef srcCol = decideSourceData(model, col); TableSignature inputSig = inpTable.getSignature(); if (inputSig == null) // table does not exists @@ -387,8 +374,7 @@ public class DictionaryManager { private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { final ResourceStore store = MetadataManager.getInstance(config).getStore(); - final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, - DictionaryInfoSerializer.INFO_SERIALIZER); + final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); TableSignature input = dictInfo.getInput(); @@ -402,8 +388,7 @@ public class DictionaryManager { private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException { final ResourceStore store = MetadataManager.getInstance(config).getStore(); - final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, - DictionaryInfoSerializer.INFO_SERIALIZER); + final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); DictionaryInfo largestDict = null; for (DictionaryInfo dictionaryInfo : allResources) { @@ -459,10 +444,8 @@ public class DictionaryManager { DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); - logger.info("DictionaryManager(" + System.identityHashCode(this) + ") loading DictionaryInfo(loadDictObj:" - + loadDictObj + ") at " + resourcePath); - DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, - loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER); + logger.info("DictionaryManager(" + System.identityHashCode(this) + ") loading DictionaryInfo(loadDictObj:" + loadDictObj + ") at " + resourcePath); + DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER); return info; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java index 002a89a..a593371 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java @@ -63,8 +63,7 @@ public class GlobalDictionaryBuilder implements IDictionaryBuilder { if (lock.lock(getLockPath(sourceColumn))) { logger.info("processed {} values for {}", counter, sourceColumn); } else { - throw new RuntimeException( - "Failed to create global dictionary on " + sourceColumn + " This client doesn't keep the lock"); + throw new RuntimeException("Failed to create global dictionary on " + sourceColumn + " This client doesn't keep the lock"); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java index 72491a3..0934a7d 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java @@ -29,10 +29,10 @@ public interface IDictionaryBuilder { /** Sets the dictionary info for the dictionary being built. Mainly for GlobalDictionaryBuilder. */ void init(DictionaryInfo info, int baseId) throws IOException; - + /** Add a new value into dictionary, returns it is accepted (not null) or not. */ boolean addValue(String value); - + /** Build the dictionary */ Dictionary<String> build() throws IOException; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java index cbd9108..f193d4f 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java @@ -24,7 +24,7 @@ import java.io.IOException; * Created by dongli on 10/28/15. */ public interface IDictionaryValueEnumerator { - + String current() throws IOException; boolean moveNext() throws IOException; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java index 027b3dc..5c80a26 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java @@ -31,7 +31,7 @@ public class IterableDictionaryValueEnumerator implements IDictionaryValueEnumer public IterableDictionaryValueEnumerator(String... strs) { this(Arrays.asList(strs)); } - + public IterableDictionaryValueEnumerator(Iterable<String> list) { iterator = list.iterator(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java index 33ed607..c1bc5d5 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java @@ -57,7 +57,7 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer while (curDictIndex < dictionaryList.size()) { if (curKey <= curDict.getMaxId()) { curValue = curDict.getValueFromId(curKey); - curKey++; + curKey ++; return true; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java b/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java index 4df0c84..397ca9f 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java @@ -46,7 +46,7 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab return codec; } - public Number2BytesConverter() { + public Number2BytesConverter(){ this.maxDigitsBeforeDecimalPoint = MAX_DIGITS_BEFORE_DECIMAL_POINT; } @@ -74,11 +74,11 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab if (badBegin) { v = new BigDecimal(v).toPlainString(); } - + while (v.contains(".") && (v.endsWith("0") || v.endsWith("."))) { v = v.substring(0, v.length() - 1); } - + return v; } @@ -117,9 +117,9 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab return; } + if (len > buf.length) { - throw new IllegalArgumentException("Too many digits for NumberDictionary: " - + Bytes.toString(value, offset, len) + ". Internal buffer is only " + buf.length + " bytes"); + throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Internal buffer is only " + buf.length + " bytes"); } boolean negative = value[offset] == '-'; @@ -149,9 +149,7 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab // prepend '0' int nZeroPadding = maxDigitsBeforeDecimalPoint - (decimalPoint - start); if (nZeroPadding < 0 || nZeroPadding + 1 > start) - throw new IllegalArgumentException( - "Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Expect " - + maxDigitsBeforeDecimalPoint + " digits before decimal point at max."); + throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Expect " + maxDigitsBeforeDecimalPoint + " digits before decimal point at max."); for (int i = 0; i < nZeroPadding; i++) { buf[--start] = '0'; } @@ -243,4 +241,4 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab return out - offset; } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java index 89b4701..1377e8e 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java @@ -18,6 +18,7 @@ package org.apache.kylin.dict; + import org.apache.kylin.common.util.ClassUtil; /** @@ -28,6 +29,7 @@ import org.apache.kylin.common.util.ClassUtil; @Deprecated public class NumberDictionary<T> extends TrieDictionary<T> { + // ============================================================================ public NumberDictionary() { // default constructor for Writable interface @@ -48,8 +50,8 @@ public class NumberDictionary<T> extends TrieDictionary<T> { protected void setConverterByName(String converterName) throws Exception { converterName = "org.apache.kylin.dict.Number2BytesConverter"; this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance(); - ((Number2BytesConverter) this.bytesConvert) - .setMaxDigitsBeforeDecimalPoint(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT_LEGACY); + ((Number2BytesConverter)this.bytesConvert).setMaxDigitsBeforeDecimalPoint(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT_LEGACY); } -} + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java index f7edf88..26e4f89 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java @@ -18,6 +18,7 @@ package org.apache.kylin.dict; + /** * Use <code>NumberDictionaryForestBuilder</code> instead. * @@ -26,10 +27,12 @@ package org.apache.kylin.dict; @Deprecated public class NumberDictionaryBuilder extends TrieDictionaryBuilder<String> { + public NumberDictionaryBuilder() { super(new Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT)); } + public NumberDictionary build(int baseId) { byte[] trieBytes = buildTrieBytes(baseId); NumberDictionary2 r = new NumberDictionary2(trieBytes); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java index afd013b..380cd1d 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java @@ -41,12 +41,10 @@ public class NumberDictionaryForestBuilder extends TrieDictionaryForestBuilder<S } public NumberDictionaryForestBuilder(int baseId) { - super(new org.apache.kylin.dict.Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT), - 0); + super(new org.apache.kylin.dict.Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT), 0); } public NumberDictionaryForestBuilder(int baseId, int maxTrieSizeMB) { - super(new org.apache.kylin.dict.Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT), 0, - maxTrieSizeMB); + super(new org.apache.kylin.dict.Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT), 0, maxTrieSizeMB); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java index 960681a..7caf686 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java @@ -47,8 +47,7 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator { } else { // normal case if (split.length <= colIndex) { - throw new ArrayIndexOutOfBoundsException( - "Column no. " + colIndex + " not found, line split is " + Arrays.asList(split)); + throw new ArrayIndexOutOfBoundsException("Column no. " + colIndex + " not found, line split is " + Arrays.asList(split)); } colStrValue = split[colIndex]; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java index ff4cc64..8f9f74f 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java @@ -41,8 +41,7 @@ public class TableColumnValueSortedEnumerator implements IDictionaryValueEnumera private PriorityQueue<ReaderBuffer> pq; - public TableColumnValueSortedEnumerator(Collection<IReadableTable.TableReader> readers, int colIndex, - final Comparator<String> comparator) { + public TableColumnValueSortedEnumerator(Collection<IReadableTable.TableReader> readers, int colIndex, final Comparator<String> comparator) { this.readers = readers; this.colIndex = colIndex; this.comparator = comparator; @@ -90,6 +89,7 @@ public class TableColumnValueSortedEnumerator implements IDictionaryValueEnumera return false; } + @Override public void close() throws IOException { for (IReadableTable.TableReader reader : readers) { @@ -136,8 +136,7 @@ public class TableColumnValueSortedEnumerator implements IDictionaryValueEnumera } else { // normal case if (split.length <= colIndex) { - throw new ArrayIndexOutOfBoundsException( - "Column no. " + colIndex + " not found, line split is " + Arrays.asList(split)); + throw new ArrayIndexOutOfBoundsException("Column no. " + colIndex + " not found, line split is " + Arrays.asList(split)); } this.cache = split[colIndex]; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java index 160bfc5..8849015 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java @@ -29,6 +29,7 @@ import java.io.ObjectOutputStream; import java.io.PrintStream; import java.util.Arrays; + import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.common.util.ClassUtil; @@ -51,11 +52,11 @@ import com.google.common.base.Preconditions; * * @author yangli9 */ -@SuppressWarnings({ "rawtypes", "unchecked" }) +@SuppressWarnings({"rawtypes", "unchecked"}) public class TrieDictionary<T> extends CacheDictionary<T> { private static final long serialVersionUID = 1L; - public static final byte[] MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict" + public static final byte[] MAGIC = new byte[]{0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74}; // "TrieDict" public static final int MAGIC_SIZE_I = MAGIC.length; public static final int BIT_IS_LAST_CHILD = 0x80; @@ -78,6 +79,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> { transient private long childOffsetMask; transient private int firstByteOffset; + public TrieDictionary() { // default constructor for Writable interface } @@ -141,6 +143,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> { return maxValueLength; } + @Override protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) { int seq = lookupSeqNoFromValue(headSize, value, offset, offset + len, roundingFlag); @@ -230,6 +233,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> { return k; } + @Override protected byte[] getValueBytesFromIdWithoutCache(int id) { byte[] buf = new byte[maxValueLength]; @@ -409,8 +413,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> { ByteArrayOutputStream baos = new ByteArrayOutputStream(); new ObjectOutputStream(baos).writeObject(dict); - TrieDictionary<String> dict2 = (TrieDictionary<String>) new ObjectInputStream( - new ByteArrayInputStream(baos.toByteArray())).readObject(); + TrieDictionary<String> dict2 = (TrieDictionary<String>) new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())).readObject(); Preconditions.checkArgument(dict.contains(dict2)); Preconditions.checkArgument(dict2.contains(dict)); Preconditions.checkArgument(dict.equals(dict2)); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java index 12ef117..1750ac1 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java @@ -216,10 +216,7 @@ public class TrieDictionaryBuilder<T> { // flatten trie footprint calculation, case of One-Byte-Per-Node out.println("----------------------------------------------------------------------------"); - out.println("OBPN node size: " - + (obpn_sizeValue + obpn_sizeNoValuesBeneath + obpn_sizeChildCount + obpn_sizeChildOffset) + " = " - + obpn_sizeValue + " + " + obpn_sizeNoValuesBeneath + " + " + obpn_sizeChildCount + " + " - + obpn_sizeChildOffset); + out.println("OBPN node size: " + (obpn_sizeValue + obpn_sizeNoValuesBeneath + obpn_sizeChildCount + obpn_sizeChildOffset) + " = " + obpn_sizeValue + " + " + obpn_sizeNoValuesBeneath + " + " + obpn_sizeChildCount + " + " + obpn_sizeChildOffset); out.println("OBPN no. nodes: " + obpn_nNodes); out.println("OBPN trie depth: " + maxValueLength); out.println("OBPN footprint: " + obpn_footprint + " in bytes"); @@ -231,9 +228,7 @@ public class TrieDictionaryBuilder<T> { out.println("MBPN total fan out: " + mbpn_nTotalFanOut); out.println("MBPN average fan out: " + (double) mbpn_nTotalFanOut / mbpn_nChildLookups); out.println("MBPN values size total: " + mbpn_sizeValueTotal); - out.println("MBPN node size: " - + (mbpn_sizeNoValueBytes + mbpn_sizeNoValueBeneath + mbpn_sizeChildOffset) + " = " - + mbpn_sizeNoValueBytes + " + " + mbpn_sizeNoValueBeneath + " + " + mbpn_sizeChildOffset); + out.println("MBPN node size: " + (mbpn_sizeNoValueBytes + mbpn_sizeNoValueBeneath + mbpn_sizeChildOffset) + " = " + mbpn_sizeNoValueBytes + " + " + mbpn_sizeNoValueBeneath + " + " + mbpn_sizeChildOffset); out.println("MBPN no. nodes: " + mbpn_nNodes); out.println("MBPN trie depth: " + mbpn_trieDepth); out.println("MBPN footprint: " + mbpn_footprint + " in bytes"); @@ -297,11 +292,9 @@ public class TrieDictionaryBuilder<T> { s.obpn_sizeChildCount = 1; s.obpn_sizeChildOffset = 5; // MSB used as isEndOfValue flag s.obpn_nNodes = s.nValueBytesCompressed; // no. nodes is the total number of compressed bytes in OBPN - s.obpn_footprint = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount - + s.obpn_sizeChildOffset); + s.obpn_footprint = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset); while (true) { // minimize the offset size to match the footprint - long t = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount - + s.obpn_sizeChildOffset - 1); + long t = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset - 1); if (BytesUtil.sizeForValue(t * 2) <= s.obpn_sizeChildOffset - 1) { // *2 because MSB of offset is used for isEndOfValue flag s.obpn_sizeChildOffset--; s.obpn_footprint = t; @@ -314,11 +307,9 @@ public class TrieDictionaryBuilder<T> { s.mbpn_sizeNoValueBytes = 1; s.mbpn_sizeNoValueBeneath = BytesUtil.sizeForValue(s.nValues); s.mbpn_sizeChildOffset = 5; - s.mbpn_footprint = s.mbpn_sizeValueTotal - + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset); + s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset); while (true) { // minimize the offset size to match the footprint - long t = s.mbpn_sizeValueTotal + s.mbpn_nNodes - * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1); + long t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1); if (BytesUtil.sizeForValue(t * 4) <= s.mbpn_sizeChildOffset - 1) { // *4 because 2 MSB of offset is used for isEndOfValue & isEndChild flag s.mbpn_sizeChildOffset--; s.mbpn_footprint = t; @@ -493,14 +484,12 @@ public class TrieDictionaryBuilder<T> { } private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) { - int flags = (int) trieBytes[parentOffset] - & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE); + int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE); BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset); trieBytes[parentOffset] |= flags; } - private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, - byte[] trieBytes) { + private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) { int o = offset; if (o > _2GB) throw new IllegalStateException(); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java index 6ef767b..09d5bc2 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java @@ -63,12 +63,11 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> { } public TrieDictionaryForest(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, // - ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) { + ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) { init(trees, valueDivide, accuOffset, bytesConverter, baseId); } - private void init(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, - ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) { + private void init(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) { this.trees = trees; this.valueDivide = valueDivide; this.accuOffset = accuOffset; @@ -99,8 +98,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> { } @Override - protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) - throws IllegalArgumentException { + protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException { int index; if (trees.size() == 1) { index = 0; @@ -111,8 +109,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> { if (roundingFlag > 0) { return getMinId(); //searching value smaller than the smallest value in dict } else { - throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" - + Bytes.toStringBinary(value, offset, len) + ") not exists!"); + throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!"); } } @@ -121,15 +118,13 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> { if (search.compareTo(maxValueOfTree) > 0) index++; if (index >= trees.size()) - throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" - + Bytes.toStringBinary(value, offset, len) + ") not exists!"); + throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!"); } } TrieDictionary<T> tree = trees.get(index); int id = tree.getIdFromValueBytesWithoutCache(value, offset, len, roundingFlag); if (id == -1) - throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" - + Bytes.toStringBinary(value, offset, len) + ") not exists!"); + throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!"); id = id + accuOffset.get(index); id += baseId; return id; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java index 482c12a..0e5e63e 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java @@ -17,13 +17,13 @@ */ package org.apache.kylin.dict; -import java.util.ArrayList; - import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ByteArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; + /** * Build a trie dictionary forest if the input values is ordered, or the forest falls back to a single trie. */ @@ -86,12 +86,10 @@ public class TrieDictionaryForestBuilder<T> { return; //duplicate value } if (comp > 0) { - logger.info("values not in ascending order, previous '{}', current '{}'", previousValue, - valueByteArray); + logger.info("values not in ascending order, previous '{}', current '{}'", previousValue, valueByteArray); isOrdered = false; if (trees.size() > 0) { - throw new IllegalStateException( - "Invalid input data. Unordered data cannot be split into multi trees"); + throw new IllegalStateException("Invalid input data. Unordered data cannot be split into multi trees"); } } } @@ -112,8 +110,7 @@ public class TrieDictionaryForestBuilder<T> { addTree(tree); reset(); } - TrieDictionaryForest<T> forest = new TrieDictionaryForest<T>(this.trees, this.valueDivide, this.accuOffset, - this.bytesConverter, baseId); + TrieDictionaryForest<T> forest = new TrieDictionaryForest<T>(this.trees, this.valueDivide, this.accuOffset, this.bytesConverter, baseId); // if input values are not in ascending order and tree num>1,TrieDictionaryForest can not work correctly. if (forest.getTrees().size() > 1 && !isOrdered) { throw new IllegalStateException("Invalid input data. Unordered data can not be split into multi trees"); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java index 31e938a..ee3a2c2 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java @@ -18,6 +18,11 @@ package org.apache.kylin.dict.global; +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.dict.AppendTrieDictionary; +import org.apache.kylin.dict.TrieDictionary; + import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -27,11 +32,6 @@ import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.LinkedList; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.dict.AppendTrieDictionary; -import org.apache.kylin.dict.TrieDictionary; - public class AppendDictNode { public byte[] part; public int id = -1; @@ -195,14 +195,12 @@ public class AppendDictNode { } private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) { - int flags = (int) trieBytes[parentOffset] - & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE); + int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE); BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset); trieBytes[parentOffset] |= flags; } - private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId, - byte[] trieBytes) { + private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId, byte[] trieBytes) { int o = offset; // childOffset @@ -214,8 +212,7 @@ public class AppendDictNode { // nValueBytes if (n.part.length > 255) - throw new RuntimeException( - "Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part)); + throw new RuntimeException("Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part)); BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1); o++; @@ -241,8 +238,7 @@ public class AppendDictNode { @Override public String toString() { - return String.format("DictNode[root=%s, nodes=%d, firstValue=%s]", Bytes.toStringBinary(part), childrenCount, - Bytes.toStringBinary(firstValue())); + return String.format("DictNode[root=%s, nodes=%d, firstValue=%s]", Bytes.toStringBinary(part), childrenCount, Bytes.toStringBinary(firstValue())); } static class Stats { @@ -336,8 +332,7 @@ public class AppendDictNode { s.mbpn_sizeValueTotal = s.nValueBytesCompressed + s.nValues * s.mbpn_sizeId; s.mbpn_sizeNoValueBytes = 1; s.mbpn_sizeChildOffset = 5; - s.mbpn_footprint = s.mbpn_sizeValueTotal - + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset); + s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset); while (true) { // minimize the offset size to match the footprint int t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset - 1); // *4 because 2 MSB of offset is used for isEndOfValue & isEndChild flag @@ -380,4 +375,4 @@ public class AppendDictNode { }, 0); } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java index a21832d..4e820e0 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java @@ -18,6 +18,9 @@ package org.apache.kylin.dict.global; +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.BytesUtil; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInput; @@ -26,12 +29,8 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashSet; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; - public class AppendDictSlice { - static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44, - 0x69, 0x63, 0x74 }; // "AppendTrieDict" + static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "AppendTrieDict" static final int HEAD_SIZE_I = HEAD_MAGIC.length; static final int BIT_IS_LAST_CHILD = 0x80; static final int BIT_IS_END_OF_VALUE = 0x40; @@ -59,8 +58,7 @@ public class AppendDictSlice { throw new IllegalArgumentException("Wrong file type (magic does not match)"); try { - DataInputStream headIn = new DataInputStream( - new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I)); + DataInputStream headIn = new DataInputStream(new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I)); this.headSize = headIn.readShort(); this.bodyLen = headIn.readInt(); this.nValues = headIn.readInt(); @@ -106,8 +104,7 @@ public class AppendDictSlice { if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) { break; } - nodeOffset = headSize - + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask); + nodeOffset = headSize + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask); if (nodeOffset == headSize) { break; } @@ -158,8 +155,7 @@ public class AppendDictSlice { } else if (comp < 0) { // try next child if (checkFlag(c, BIT_IS_LAST_CHILD)) return -1; - c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1) - + (checkFlag(c, BIT_IS_END_OF_VALUE) ? sizeOfId : 0); + c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1) + (checkFlag(c, BIT_IS_END_OF_VALUE) ? sizeOfId : 0); } else { // children are ordered by their first value byte return -1; } @@ -265,8 +261,7 @@ public class AppendDictSlice { @Override public String toString() { - return String.format("DictSlice[firstValue=%s, values=%d, bytes=%d]", Bytes.toStringBinary(getFirstValue()), - nValues, bodyLen); + return String.format("DictSlice[firstValue=%s, values=%d, bytes=%d]", Bytes.toStringBinary(getFirstValue()), nValues, bodyLen); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java index d61258a..323fe6b 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java @@ -18,13 +18,13 @@ package org.apache.kylin.dict.global; +import org.apache.kylin.common.util.Bytes; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.Arrays; -import org.apache.kylin.common.util.Bytes; - public class AppendDictSliceKey implements Comparable<AppendDictSliceKey> { static final AppendDictSliceKey START_KEY = AppendDictSliceKey.wrap(new byte[0]); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java index 2911798..90d65b6 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java @@ -18,18 +18,18 @@ package org.apache.kylin.dict.global; -import static com.google.common.base.Preconditions.checkState; - -import java.io.IOException; -import java.util.List; -import java.util.TreeMap; - import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.dict.AppendTrieDictionary; import org.apache.kylin.dict.BytesConverter; import org.apache.kylin.dict.StringBytesConverter; +import java.io.IOException; +import java.util.List; +import java.util.TreeMap; + +import static com.google.common.base.Preconditions.checkState; + public class AppendTrieDictionaryBuilder { private final String baseDir; @@ -47,8 +47,7 @@ public class AppendTrieDictionaryBuilder { private AppendDictNode curNode; public AppendTrieDictionaryBuilder(String resourceDir, int maxEntriesPerSlice) throws IOException { - this.baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + resourceDir - + "/"; + this.baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + resourceDir + "/"; this.workingDir = this.baseDir + "/working"; this.maxEntriesPerSlice = maxEntriesPerSlice; init(); @@ -84,8 +83,7 @@ public class AppendTrieDictionaryBuilder { curNode = new AppendDictNode(new byte[0], false); sliceFileMap.put(AppendDictSliceKey.START_KEY, null); } - checkState(sliceFileMap.firstKey().equals(AppendDictSliceKey.START_KEY), - "first key should be \"\", but got \"%s\"", sliceFileMap.firstKey()); + checkState(sliceFileMap.firstKey().equals(AppendDictSliceKey.START_KEY), "first key should be \"\", but got \"%s\"", sliceFileMap.firstKey()); AppendDictSliceKey nextKey = sliceFileMap.floorKey(AppendDictSliceKey.wrap(valueBytes)); @@ -123,8 +121,7 @@ public class AppendTrieDictionaryBuilder { flushCurrentNode(); } - GlobalDictMetadata metadata = new GlobalDictMetadata(baseId, this.maxId, this.maxValueLength, this.nValues, - this.bytesConverter, sliceFileMap); + GlobalDictMetadata metadata = new GlobalDictMetadata(baseId, this.maxId, this.maxValueLength, this.nValues, this.bytesConverter, sliceFileMap); store.commit(workingDir, metadata); AppendTrieDictionary dict = new AppendTrieDictionary(); @@ -161,8 +158,7 @@ public class AppendTrieDictionaryBuilder { } } else { // otherwise, split the current node into two - AppendDictNode c = new AppendDictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, - node.children); + AppendDictNode c = new AppendDictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children); c.id = node.id; node.reset(BytesUtil.subarray(node.part, 0, i), true); node.addChild(c); @@ -174,8 +170,7 @@ public class AppendTrieDictionaryBuilder { // if partially matched the current, split the current node, add the new // value, make a 3-way if (i < n) { - AppendDictNode c1 = new AppendDictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, - node.children); + AppendDictNode c1 = new AppendDictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children); c1.id = node.id; AppendDictNode c2 = addNodeMaybeOverflow(value, j, nn); node.reset(BytesUtil.subarray(node.part, 0, i), false);
