http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java index f8ed82b..75036c0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java @@ -29,23 +29,18 @@ public class CubeMetadataUpgradeEntry_v_1_5_1 { public static void main(String[] args) { if (!(args != null && (args.length == 1))) { System.out.println("Usage: java CubeMetadataUpgradeEntry_v_1_5_1 <metadata_export_folder>"); - System.out.println( - ", where metadata_export_folder is the folder containing your current metadata's dump (Upgrade program will not modify it directly, relax."); + System.out.println(", where metadata_export_folder is the folder containing your current metadata's dump (Upgrade program will not modify it directly, relax."); return; } try { CubeMetadataUpgrade_v_1_4_0.upgradeOrVerify(CubeMetadataUpgrade_v_1_4_0.class, args, true, false); - CubeMetadataUpgrade_v_1_5_1.upgradeOrVerify(CubeMetadataUpgrade_v_1_5_1.class, - new String[] { args[0] + "_workspace" }, false, true); + CubeMetadataUpgrade_v_1_5_1.upgradeOrVerify(CubeMetadataUpgrade_v_1_5_1.class, new String[] { args[0] + "_workspace" }, false, true); } catch (Exception e) { - logger.error( - "something went wrong when upgrading, don't override your metadata store with this workspace folder yet!", - e); + logger.error("something went wrong when upgrading, don't override your metadata store with this workspace folder yet!", e); return; } - logger.info( - "The metadata upgrade is complete locally. You need to upload the metadata to you actual metadata store to verify locally. You need to upload the metadata to you actual metadata store to verify."); + logger.info("The metadata upgrade is complete locally. You need to upload the metadata to you actual metadata store to verify locally. You need to upload the metadata to you actual metadata store to verify."); } }
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java index 02ba81a..587842a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java @@ -112,8 +112,7 @@ public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade { if (!modelDimMap.containsKey(cubeDimDesc.getTable())) { modelDimMap.put(cubeDimDesc.getTable(), new HashSet<String>()); } - modelDimMap.get(cubeDimDesc.getTable()).addAll(Lists.newArrayList( - cubeDimDesc.getDerived() != null ? cubeDimDesc.getDerived() : cubeDimDesc.getColumn())); + modelDimMap.get(cubeDimDesc.getTable()).addAll(Lists.newArrayList(cubeDimDesc.getDerived() != null ? cubeDimDesc.getDerived() : cubeDimDesc.getColumn())); } List<ModelDimensionDesc> modelDimDescList = Lists.newArrayListWithCapacity(modelDimMap.size()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java index 78fbcd4..b1b6bce 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java @@ -59,8 +59,7 @@ public class CubingUtils { private static Logger logger = LoggerFactory.getLogger(CubingUtils.class); - public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, - Iterable<List<String>> streams) { + public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) { final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc); final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length; final List<Long> allCuboidIds = new CuboidScheduler(cubeDesc).getAllCuboidIds(); @@ -133,10 +132,8 @@ public class CubingUtils { return result; } - public static Map<TblColRef, Dictionary<String>> buildDictionary(final CubeInstance cubeInstance, - Iterable<List<String>> recordList) throws IOException { - final List<TblColRef> columnsNeedToBuildDictionary = cubeInstance.getDescriptor() - .listDimensionColumnsExcludingDerived(true); + public static Map<TblColRef, Dictionary<String>> buildDictionary(final CubeInstance cubeInstance, Iterable<List<String>> recordList) throws IOException { + final List<TblColRef> columnsNeedToBuildDictionary = cubeInstance.getDescriptor().listDimensionColumnsExcludingDerived(true); final HashMap<Integer, TblColRef> tblColRefMap = Maps.newHashMap(); int index = 0; for (TblColRef column : columnsNeedToBuildDictionary) { @@ -156,16 +153,14 @@ public class CubingUtils { } for (TblColRef tblColRef : valueMap.keySet()) { Set<String> values = valueMap.get(tblColRef); - Dictionary<String> dict = DictionaryGenerator.buildDictionary(tblColRef.getType(), - new IterableDictionaryValueEnumerator(values)); + Dictionary<String> dict = DictionaryGenerator.buildDictionary(tblColRef.getType(), new IterableDictionaryValueEnumerator(values)); result.put(tblColRef, dict); } return result; } @SuppressWarnings("unchecked") - public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, - Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { + public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java index 2ebec04..92d0fac 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java @@ -272,8 +272,7 @@ public class GTAggregateScanner implements IGTScanner { final long estMemSize = estimatedMemSize(); if (spillThreshold > 0 && estMemSize > spillThreshold) { if (!spillEnabled) { - throw new ResourceLimitExceededException("aggregation's memory consumption " + estMemSize - + " exceeds threshold " + spillThreshold); + throw new ResourceLimitExceededException("aggregation's memory consumption " + estMemSize + " exceeds threshold " + spillThreshold); } spillBuffMap(estMemSize); // spill to disk aggBufMap = createBuffMap(); @@ -357,8 +356,7 @@ public class GTAggregateScanner implements IGTScanner { final ReturningRecord returningRecord = new ReturningRecord(); Entry<byte[], MeasureAggregator[]> returningEntry = null; - final HavingFilterChecker havingFilterChecker = (havingFilter == null) ? null - : new HavingFilterChecker(); + final HavingFilterChecker havingFilterChecker = (havingFilter == null) ? null : new HavingFilterChecker(); @Override public boolean hasNext() { @@ -532,8 +530,7 @@ public class GTAggregateScanner implements IGTScanner { public Iterator<Pair<byte[], byte[]>> iterator() { try { if (dumpedFile == null || !dumpedFile.exists()) { - throw new RuntimeException("Dumped file cannot be found at: " - + (dumpedFile == null ? "<null>" : dumpedFile.getAbsolutePath())); + throw new RuntimeException("Dumped file cannot be found at: " + (dumpedFile == null ? "<null>" : dumpedFile.getAbsolutePath())); } dis = new DataInputStream(new FileInputStream(dumpedFile)); @@ -558,8 +555,7 @@ public class GTAggregateScanner implements IGTScanner { dis.read(value); return new Pair<>(key, value); } catch (Exception e) { - throw new RuntimeException( - "Cannot read AggregationCache from dumped file: " + e.getMessage()); + throw new RuntimeException("Cannot read AggregationCache from dumped file: " + e.getMessage()); } } @@ -574,8 +570,7 @@ public class GTAggregateScanner implements IGTScanner { } public void flush() throws IOException { - logger.info("AggregationCache(size={} est_mem_size={} threshold={}) will spill to {}", buffMap.size(), - estMemSize, spillThreshold, dumpedFile.getAbsolutePath()); + logger.info("AggregationCache(size={} est_mem_size={} threshold={}) will spill to {}", buffMap.size(), estMemSize, spillThreshold, dumpedFile.getAbsolutePath()); if (buffMap != null) { DataOutputStream dos = null; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java index df9a234..ba62af3 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java @@ -294,8 +294,7 @@ public class GTInfo { BytesUtil.writeAsciiString(value.codeSystem.getClass().getCanonicalName(), out); BytesSerializer<IGTCodeSystem> serializer = null; try { - serializer = (BytesSerializer<IGTCodeSystem>) value.codeSystem.getClass().getField("serializer") - .get(null); + serializer = (BytesSerializer<IGTCodeSystem>) value.codeSystem.getClass().getField("serializer").get(null); } catch (IllegalAccessException | NoSuchFieldException e) { throw new RuntimeException("failed to get serializer for " + value.codeSystem.getClass(), e); } @@ -327,8 +326,7 @@ public class GTInfo { } else { try { Class clazz = Class.forName(codeSystemType); - BytesSerializer<IGTCodeSystem> serializer = (BytesSerializer<IGTCodeSystem>) clazz - .getField("serializer").get(null); + BytesSerializer<IGTCodeSystem> serializer = (BytesSerializer<IGTCodeSystem>) clazz.getField("serializer").get(null); codeSystem = serializer.deserialize(in); } catch (Exception e) { throw new RuntimeException("Failed to deserialize IGTCodeSystem " + codeSystemType, e); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java index 97d9f53..3e62ea7 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java @@ -45,7 +45,7 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable { } this.info = info; } - + @Override public GTRecord clone() { // deep copy ByteArray[] cols = new ByteArray[this.cols.length]; @@ -114,6 +114,7 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable { return result; } + /** decode and return the values of this record */ public Object[] getValues(int[] selectedColumns, Object[] result) { assert selectedColumns.length <= result.length; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java index 804d06d..2cbbd8d 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java @@ -94,8 +94,7 @@ public class GTRowBlock { primaryKey.copyFrom(other.primaryKey); for (int i = 0; i < info.colBlocks.length; i++) { cellBlockBuffers[i].clear(); - cellBlockBuffers[i].put(other.cellBlocks[i].array(), other.cellBlocks[i].offset(), - other.cellBlocks[i].length()); + cellBlockBuffers[i].put(other.cellBlocks[i].array(), other.cellBlocks[i].offset(), other.cellBlocks[i].length()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 1a89471..ffaa8bd 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -78,8 +78,7 @@ public class GTScanRequest { private transient boolean doingStorageAggregation = false; GTScanRequest(GTInfo info, List<GTScanRange> ranges, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, // - ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, TupleFilter filterPushDown, - TupleFilter havingFilterPushDown, // + ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, TupleFilter filterPushDown, TupleFilter havingFilterPushDown, // boolean allowStorageAggregation, double aggCacheMemThreshold, int storageScanRowNumThreshold, // int storagePushDownLimit, String storageBehavior, long startTime, long timeout) { this.info = info; @@ -173,16 +172,14 @@ public class GTScanRequest { * * Refer to CoprocessorBehavior for explanation */ - public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn) - throws IOException { + public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn) throws IOException { return decorateScanner(scanner, filterToggledOn, aggrToggledOn, false, true); } /** * hasPreFiltered indicate the data has been filtered before scanning */ - public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, - boolean hasPreFiltered, boolean spillEnabled) throws IOException { + public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, boolean spillEnabled) throws IOException { IGTScanner result = scanner; if (!filterToggledOn) { //Skip reading this section if you're not profiling! lookAndForget(result); @@ -283,7 +280,7 @@ public class GTScanRequest { public TupleFilter getHavingFilterPushDown() { return havingFilterPushDown; } - + public ImmutableBitSet getDimensions() { return this.getColumns().andNot(this.getAggrMetrics()); } @@ -337,9 +334,7 @@ public class GTScanRequest { @Override public String toString() { - return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown - + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" - + Arrays.toString(aggrMetricsFuncs) + "]"; + return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]"; } public byte[] toByteArray() { @@ -369,8 +364,7 @@ public class GTScanRequest { ImmutableBitSet.serializer.serialize(value.columns, out); BytesUtil.writeByteArray(GTUtil.serializeGTFilter(value.filterPushDown, value.info), out); - BytesUtil.writeByteArray( - TupleFilterSerializer.serialize(value.havingFilterPushDown, StringCodeSystem.INSTANCE), out); + BytesUtil.writeByteArray(TupleFilterSerializer.serialize(value.havingFilterPushDown, StringCodeSystem.INSTANCE), out); ImmutableBitSet.serializer.serialize(value.aggrGroupBy, out); ImmutableBitSet.serializer.serialize(value.aggrMetrics, out); @@ -404,8 +398,7 @@ public class GTScanRequest { ImmutableBitSet sColumns = ImmutableBitSet.serializer.deserialize(in); TupleFilter sGTFilter = GTUtil.deserializeGTFilter(BytesUtil.readByteArray(in), sInfo); - TupleFilter sGTHavingFilter = TupleFilterSerializer.deserialize(BytesUtil.readByteArray(in), - StringCodeSystem.INSTANCE); + TupleFilter sGTHavingFilter = TupleFilterSerializer.deserialize(BytesUtil.readByteArray(in), StringCodeSystem.INSTANCE); ImmutableBitSet sAggGroupBy = ImmutableBitSet.serializer.deserialize(in); ImmutableBitSet sAggrMetrics = ImmutableBitSet.serializer.deserialize(in); @@ -420,8 +413,7 @@ public class GTScanRequest { return new GTScanRequestBuilder().setInfo(sInfo).setRanges(sRanges).setDimensions(sColumns).// setAggrGroupBy(sAggGroupBy).setAggrMetrics(sAggrMetrics).setAggrMetricsFuncs(sAggrMetricFuncs).// - setFilterPushDown(sGTFilter).setHavingFilterPushDown(sGTHavingFilter) - .setAllowStorageAggregation(sAllowPreAggr).setAggCacheMemThreshold(sAggrCacheGB).// + setFilterPushDown(sGTFilter).setHavingFilterPushDown(sGTHavingFilter).setAllowStorageAggregation(sAllowPreAggr).setAggCacheMemThreshold(sAggrCacheGB).// setStorageScanRowNumThreshold(storageScanRowNumThreshold).setStoragePushDownLimit(storagePushDownLimit).// setStartTime(startTime).setTimeout(timeout).setStorageBehavior(storageBehavior).createGTScanRequest(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java index fe09061..ba1fdbc 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java @@ -61,7 +61,7 @@ public class GTScanRequestBuilder { this.havingFilterPushDown = havingFilterPushDown; return this; } - + public GTScanRequestBuilder setDimensions(ImmutableBitSet dimensions) { this.dimensions = dimensions; return this; @@ -131,16 +131,12 @@ public class GTScanRequestBuilder { } if (storageBehavior == null) { - storageBehavior = BackdoorToggles.getCoprocessorBehavior() == null - ? StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() - : BackdoorToggles.getCoprocessorBehavior(); + storageBehavior = BackdoorToggles.getCoprocessorBehavior() == null ? StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior(); } this.startTime = startTime == -1 ? System.currentTimeMillis() : startTime; this.timeout = timeout == -1 ? 300000 : timeout; - return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, - havingFilterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, - storagePushDownLimit, storageBehavior, startTime, timeout); + return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, havingFilterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, storagePushDownLimit, storageBehavior, startTime, timeout); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java index 5914d09..4eb5791 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java @@ -18,18 +18,17 @@ package org.apache.kylin.gridtable; -import java.util.Comparator; -import java.util.Iterator; -import java.util.NoSuchElementException; - +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; import org.apache.kylin.GTForwardingScanner; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.measure.BufferedMeasureCodec; import org.apache.kylin.measure.MeasureAggregator; -import com.google.common.base.Preconditions; -import com.google.common.collect.Iterators; -import com.google.common.collect.PeekingIterator; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; /** * GTStreamAggregateScanner requires input records to be sorted on group fields. http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java index 900b9ff..7a7e4e6 100755 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java @@ -81,9 +81,8 @@ public class GTUtil { final Set<TblColRef> unevaluatableColumnCollector) { IFilterCodeSystem<ByteArray> filterCodeSystem = wrap(info.codeSystem.getComparator()); - - GTConvertDecorator decorator = new GTConvertDecorator(unevaluatableColumnCollector, colMapping, info, - encodeConstants); + + GTConvertDecorator decorator = new GTConvertDecorator(unevaluatableColumnCollector, colMapping, info, encodeConstants); byte[] bytes = TupleFilterSerializer.serialize(rootFilter, decorator, filterCodeSystem); return TupleFilterSerializer.deserialize(bytes, filterCodeSystem); @@ -123,15 +122,14 @@ public class GTUtil { protected final GTInfo info; protected final boolean encodeConstants; - public GTConvertDecorator(Set<TblColRef> unevaluatableColumnCollector, Map<TblColRef, Integer> colMapping, - GTInfo info, boolean encodeConstants) { + public GTConvertDecorator(Set<TblColRef> unevaluatableColumnCollector, Map<TblColRef, Integer> colMapping, GTInfo info, boolean encodeConstants) { this.unevaluatableColumnCollector = unevaluatableColumnCollector; this.colMapping = colMapping; this.info = info; this.encodeConstants = encodeConstants; buf = ByteBuffer.allocate(info.getMaxColumnLength()); } - + protected int mapCol(TblColRef col) { Integer i = colMapping.get(col); return i == null ? -1 : i; @@ -145,8 +143,7 @@ public class GTUtil { // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE, // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) // will always return FALSE. - if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT - && !TupleFilter.isEvaluableRecursively(filter)) { + if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) { TupleFilter.collectColumns(filter, unevaluatableColumnCollector); return ConstantTupleFilter.TRUE; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java index 286f3df..6190f29 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java @@ -109,18 +109,12 @@ public class UnitTestSupport { result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add( - newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add( - newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"), - new HLLCounter(14))); - result.add( - newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add( - newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add( - newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); } return result; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark.java b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark.java index 3fa3cee..589f37c 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark.java @@ -111,9 +111,7 @@ public class GTScannerBenchmark { @SuppressWarnings("unused") private void testAggregate(ImmutableBitSet groupBy) throws IOException { long t = System.currentTimeMillis(); - GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(dimensions) - .setAggrGroupBy(groupBy).setAggrMetrics(metrics).setAggrMetricsFuncs(aggrFuncs).setFilterPushDown(null) - .createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(dimensions).setAggrGroupBy(groupBy).setAggrMetrics(metrics).setAggrMetricsFuncs(aggrFuncs).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; @@ -157,8 +155,7 @@ public class GTScannerBenchmark { @SuppressWarnings("unused") private void testFilter(TupleFilter filter) throws IOException { long t = System.currentTimeMillis(); - GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(info.getAllColumns()) - .setFilterPushDown(filter).createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(info.getAllColumns()).setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java index 5302949..85d8c37 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java @@ -133,9 +133,7 @@ public class GTScannerBenchmark2 { @SuppressWarnings("unused") private void testAggregate(ImmutableBitSet groupBy) throws IOException { long t = System.currentTimeMillis(); - GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(dimensions) - .setAggrGroupBy(groupBy).setAggrMetrics(metrics).setAggrMetricsFuncs(aggrFuncs).setFilterPushDown(null) - .createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(dimensions).setAggrGroupBy(groupBy).setAggrMetrics(metrics).setAggrMetricsFuncs(aggrFuncs).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; @@ -179,8 +177,7 @@ public class GTScannerBenchmark2 { @SuppressWarnings("unused") private void testFilter(TupleFilter filter) throws IOException { long t = System.currentTimeMillis(); - GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(info.getAllColumns()) - .setFilterPushDown(filter).createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(info.getAllColumns()).setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java index dd50b1e..d81d366 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java @@ -86,8 +86,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { rule.validate(desc, vContext); vContext.print(System.out); assertTrue(vContext.getResults().length > 0); - assertTrue( - vContext.getResults()[0].getMessage().startsWith("Aggregation group 0 has too many combinations")); + assertTrue(vContext.getResults()[0].getMessage().startsWith("Aggregation group 0 has too many combinations")); } } @@ -95,9 +94,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { public void testGoodDesc2() throws IOException { ValidateContext vContext = new ValidateContext(); - CubeDesc desc = JsonUtil.readValue(new FileInputStream( - LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), - CubeDesc.class); + CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); desc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { // new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "CATEG_LVL2_NAME" } }; @@ -111,40 +108,30 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { public void testBadDesc1() throws IOException { ValidateContext vContext = new ValidateContext(); - CubeDesc desc = JsonUtil.readValue(new FileInputStream( - LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), - CubeDesc.class); - String[] temp = Arrays.asList(desc.getAggregationGroups().get(0).getIncludes()).subList(0, 3) - .toArray(new String[3]); + CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); + String[] temp = Arrays.asList(desc.getAggregationGroups().get(0).getIncludes()).subList(0, 3).toArray(new String[3]); desc.getAggregationGroups().get(0).setIncludes(temp); IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); rule.validate(desc, vContext); vContext.print(System.out); assertEquals(1, vContext.getResults().length); - assertEquals( - "Aggregation group 0 'includes' dimensions not include all the dimensions:[seller_id, META_CATEG_NAME, lstg_format_name, lstg_site_id, slr_segment_cd]", - (vContext.getResults()[0].getMessage())); + assertEquals("Aggregation group 0 'includes' dimensions not include all the dimensions:[seller_id, META_CATEG_NAME, lstg_format_name, lstg_site_id, slr_segment_cd]", (vContext.getResults()[0].getMessage())); } @Test public void testBadDesc2() throws IOException { ValidateContext vContext = new ValidateContext(); - CubeDesc desc = JsonUtil.readValue(new FileInputStream( - LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), - CubeDesc.class); + CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); desc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { // - new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "META_CATEG_NAME", - "CATEG_LVL2_NAME" } }; + new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "META_CATEG_NAME", "CATEG_LVL2_NAME" } }; IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); rule.validate(desc, vContext); vContext.print(System.out); assertEquals(1, vContext.getResults().length); - assertEquals( - "Aggregation group 0 joint dimensions has overlap with more than 1 dimensions in same hierarchy: [CATEG_LVL2_NAME, META_CATEG_NAME]", - (vContext.getResults()[0].getMessage())); + assertEquals("Aggregation group 0 joint dimensions has overlap with more than 1 dimensions in same hierarchy: [CATEG_LVL2_NAME, META_CATEG_NAME]", (vContext.getResults()[0].getMessage())); } @Test @@ -157,8 +144,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { } ValidateContext vContext = new ValidateContext(); - CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA - + "/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class); + CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class); IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); try { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index c5d89eb..0bbc874 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -110,7 +110,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { CubeDesc ic = mgr.getCubeDesc("ci_inner_join_cube"); assertNotNull(lc); assertNotNull(ic); - + // assert the two CI cubes are identical apart from the left/inner difference assertEquals(lc.getDimensions().size(), ic.getDimensions().size()); for (int i = 0, n = lc.getDimensions().size(); i < n; i++) { @@ -120,11 +120,11 @@ public class CubeDescTest extends LocalFileMetadataTestCase { assertEquals(ld.getColumn(), id.getColumn()); assertArrayEquals(ld.getDerived(), id.getDerived()); } - + // To enable spark in IT, the inner cube removed the percentile measure, so ignore that particular measure List<MeasureDesc> lcMeasures = dropPercentile(lc.getMeasures()); List<MeasureDesc> icMeasures = ic.getMeasures(); - + assertEquals(lcMeasures.size(), icMeasures.size()); for (int i = 0, n = lcMeasures.size(); i < n; i++) { MeasureDesc lm = lcMeasures.get(i); @@ -133,7 +133,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { assertEquals(lm.getFunction().getFullExpression(), im.getFunction().getFullExpression()); assertEquals(lm.getFunction().getReturnType(), im.getFunction().getReturnType()); } - + assertEquals(lc.getAggregationGroups().size(), ic.getAggregationGroups().size()); for (int i = 0, n = lc.getAggregationGroups().size(); i < n; i++) { AggregationGroup lag = lc.getAggregationGroups().get(i); @@ -143,15 +143,15 @@ public class CubeDescTest extends LocalFileMetadataTestCase { assertArrayEquals(lag.getSelectRule().hierarchyDims, iag.getSelectRule().hierarchyDims); assertArrayEquals(lag.getSelectRule().jointDims, iag.getSelectRule().jointDims); } - + assertEquals(lc.listAllColumnDescs().size(), ic.listAllColumnDescs().size()); assertEquals(lc.listAllColumns().size(), ic.listAllColumns().size()); - + // test KYLIN-2440 assertTrue(lc.listAllColumns().contains(lc.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID"))); assertTrue(ic.listAllColumns().contains(ic.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID"))); } - + private List<MeasureDesc> dropPercentile(List<MeasureDesc> measures) { ArrayList<MeasureDesc> result = new ArrayList<>(); for (MeasureDesc m : measures) { @@ -196,8 +196,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expect(IllegalStateException.class); thrown.expectMessage("Aggregation group 0 'includes' dimensions not include all the dimensions:"); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - String[] temp = Arrays.asList(cubeDesc.getAggregationGroups().get(0).getIncludes()).subList(0, 3) - .toArray(new String[3]); + String[] temp = Arrays.asList(cubeDesc.getAggregationGroups().get(0).getIncludes()).subList(0, 3).toArray(new String[3]); cubeDesc.getAggregationGroups().get(0).setIncludes(temp); cubeDesc.init(getTestConfig()); @@ -206,8 +205,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testBadInit4() throws Exception { thrown.expect(IllegalStateException.class); - thrown.expectMessage( - "Aggregation group 0 has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."); + thrown.expectMessage("Aggregation group 0 has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); try { @@ -222,8 +220,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testBadInit5() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, - META_CATEG_NAME }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, META_CATEG_NAME }; cubeDesc.init(getTestConfig()); } @@ -231,8 +228,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testBadInit6() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, - LSTG_FORMAT_NAME }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, LSTG_FORMAT_NAME }; cubeDesc.init(getTestConfig()); } @@ -243,8 +239,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expectMessage("Aggregation group 0 require at least 2 dimensions in a joint"); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { - new String[] { LSTG_FORMAT_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { LSTG_FORMAT_NAME } }; cubeDesc.init(getTestConfig()); } @@ -253,12 +248,10 @@ public class CubeDescTest extends LocalFileMetadataTestCase { public void testBadInit8() throws Exception { String[] strs = new String[] { CATEG_LVL2_NAME, META_CATEG_NAME }; thrown.expect(IllegalStateException.class); - thrown.expectMessage( - "Aggregation group 0 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); + thrown.expectMessage("Aggregation group 0 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { - new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; cubeDesc.init(getTestConfig()); } @@ -267,14 +260,10 @@ public class CubeDescTest extends LocalFileMetadataTestCase { public void testBadInit9() throws Exception { String[] strs = new String[] { LSTG_FORMAT_NAME, META_CATEG_NAME }; thrown.expect(IllegalStateException.class); - thrown.expectMessage( - "Aggregation group 0 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); + thrown.expectMessage("Aggregation group 0 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { - new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, - new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID } }; - cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { - new String[] { META_CATEG_NAME, LSTG_FORMAT_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { META_CATEG_NAME, LSTG_FORMAT_NAME } }; cubeDesc.init(getTestConfig()); } @@ -286,9 +275,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expectMessage("Aggregation group 0 a dimension exist in more than one joint: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { - new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD }, - new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, LEAF_CATEG_ID } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD }, new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, LEAF_CATEG_ID } }; cubeDesc.init(getTestConfig()); } @@ -299,8 +286,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expectMessage("Aggregation group 0 require at least 2 dimensions in a hierarchy."); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { - new String[] { META_CATEG_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME } }; cubeDesc.init(getTestConfig()); } @@ -312,9 +298,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expectMessage("Aggregation group 0 a dimension exist in more than one hierarchy: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { - new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, - new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; cubeDesc.init(getTestConfig()); } @@ -330,8 +314,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expect(IllegalStateException.class); CubeDescManager.clearCache(); - CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()) - .getCubeDesc("ut_cube_desc_combination_int_overflow"); + CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_cube_desc_combination_int_overflow"); cubeDesc.init(getTestConfig()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index 3f8e439..3cae37d 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -81,8 +81,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { CubeDescManager cubeDescMgr = getCubeDescManager(); CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); - CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc, - null); + CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); assertTrue(createdCube == cubeMgr.getCube("a_whole_new_cube")); assertTrue(prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube)); @@ -127,6 +126,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { } + @Test public void testConcurrentBuildAndMerge() throws Exception { CubeManager mgr = CubeManager.getInstance(getTestConfig()); @@ -135,30 +135,34 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { // no segment at first assertEquals(0, cube.getSegments().size()); - Map m1 = Maps.newHashMap(); + Map m1 = Maps.newHashMap(); m1.put(1, 1000L); - Map m2 = Maps.newHashMap(); + Map m2 = Maps.newHashMap(); m2.put(1, 2000L); - Map m3 = Maps.newHashMap(); + Map m3 = Maps.newHashMap(); m3.put(1, 3000L); - Map m4 = Maps.newHashMap(); + Map m4 = Maps.newHashMap(); m4.put(1, 4000L); // append first CubeSegment seg1 = mgr.appendSegment(cube, 0, 0, 0, 1000, null, m1); seg1.setStatus(SegmentStatusEnum.READY); + CubeSegment seg2 = mgr.appendSegment(cube, 0, 0, 1000, 2000, m1, m2); seg2.setStatus(SegmentStatusEnum.READY); + CubeSegment seg3 = mgr.mergeSegments(cube, 0, 0, 0000, 2000, true); seg3.setStatus(SegmentStatusEnum.NEW); + CubeSegment seg4 = mgr.appendSegment(cube, 0, 0, 2000, 3000, m2, m3); seg4.setStatus(SegmentStatusEnum.NEW); seg4.setLastBuildJobID("test"); seg4.setStorageLocationIdentifier("test"); + CubeSegment seg5 = mgr.appendSegment(cube, 0, 0, 3000, 4000, m3, m4); seg5.setStatus(SegmentStatusEnum.READY); @@ -166,23 +170,20 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { mgr.updateCube(cubeBuilder); + mgr.promoteNewlyBuiltSegments(cube, seg4); assertTrue(cube.getSegments().size() == 5); - assertTrue(cube.getSegmentById(seg1.getUuid()) != null - && cube.getSegmentById(seg1.getUuid()).getStatus() == SegmentStatusEnum.READY); - assertTrue(cube.getSegmentById(seg2.getUuid()) != null - && cube.getSegmentById(seg2.getUuid()).getStatus() == SegmentStatusEnum.READY); - assertTrue(cube.getSegmentById(seg3.getUuid()) != null - && cube.getSegmentById(seg3.getUuid()).getStatus() == SegmentStatusEnum.NEW); - assertTrue(cube.getSegmentById(seg4.getUuid()) != null - && cube.getSegmentById(seg4.getUuid()).getStatus() == SegmentStatusEnum.READY); - assertTrue(cube.getSegmentById(seg5.getUuid()) != null - && cube.getSegmentById(seg5.getUuid()).getStatus() == SegmentStatusEnum.READY); + assertTrue(cube.getSegmentById(seg1.getUuid()) != null && cube.getSegmentById(seg1.getUuid()).getStatus() == SegmentStatusEnum.READY); + assertTrue(cube.getSegmentById(seg2.getUuid()) != null && cube.getSegmentById(seg2.getUuid()).getStatus() == SegmentStatusEnum.READY); + assertTrue(cube.getSegmentById(seg3.getUuid()) != null && cube.getSegmentById(seg3.getUuid()).getStatus() == SegmentStatusEnum.NEW); + assertTrue(cube.getSegmentById(seg4.getUuid()) != null && cube.getSegmentById(seg4.getUuid()).getStatus() == SegmentStatusEnum.READY); + assertTrue(cube.getSegmentById(seg5.getUuid()) != null && cube.getSegmentById(seg5.getUuid()).getStatus() == SegmentStatusEnum.READY); } + @Test public void testConcurrentMergeAndMerge() throws Exception { System.setProperty("kylin.cube.max-building-segments", "10"); @@ -191,13 +192,13 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { // no segment at first assertEquals(0, cube.getSegments().size()); - Map m1 = Maps.newHashMap(); + Map m1 = Maps.newHashMap(); m1.put(1, 1000L); - Map m2 = Maps.newHashMap(); + Map m2 = Maps.newHashMap(); m2.put(1, 2000L); - Map m3 = Maps.newHashMap(); + Map m3 = Maps.newHashMap(); m3.put(1, 3000L); - Map m4 = Maps.newHashMap(); + Map m4 = Maps.newHashMap(); m4.put(1, 4000L); // append first @@ -213,6 +214,8 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { CubeSegment seg4 = mgr.appendSegment(cube, 0, 0, 3000, 4000, m3, m4); seg4.setStatus(SegmentStatusEnum.READY); + + CubeSegment merge1 = mgr.mergeSegments(cube, 0, 0, 0, 2000, true); merge1.setStatus(SegmentStatusEnum.NEW); merge1.setLastBuildJobID("test"); @@ -226,20 +229,17 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { CubeUpdate cubeBuilder = new CubeUpdate(cube); mgr.updateCube(cubeBuilder); + mgr.promoteNewlyBuiltSegments(cube, merge1); assertTrue(cube.getSegments().size() == 4); assertTrue(cube.getSegmentById(seg1.getUuid()) == null); assertTrue(cube.getSegmentById(seg2.getUuid()) == null); - assertTrue(cube.getSegmentById(merge1.getUuid()) != null - && cube.getSegmentById(merge1.getUuid()).getStatus() == SegmentStatusEnum.READY); - assertTrue(cube.getSegmentById(seg3.getUuid()) != null - && cube.getSegmentById(seg3.getUuid()).getStatus() == SegmentStatusEnum.READY); - assertTrue(cube.getSegmentById(seg4.getUuid()) != null - && cube.getSegmentById(seg4.getUuid()).getStatus() == SegmentStatusEnum.READY); - assertTrue(cube.getSegmentById(merge2.getUuid()) != null - && cube.getSegmentById(merge2.getUuid()).getStatus() == SegmentStatusEnum.NEW); + assertTrue(cube.getSegmentById(merge1.getUuid()) != null && cube.getSegmentById(merge1.getUuid()).getStatus() == SegmentStatusEnum.READY); + assertTrue(cube.getSegmentById(seg3.getUuid()) != null && cube.getSegmentById(seg3.getUuid()).getStatus() == SegmentStatusEnum.READY); + assertTrue(cube.getSegmentById(seg4.getUuid()) != null && cube.getSegmentById(seg4.getUuid()).getStatus() == SegmentStatusEnum.READY); + assertTrue(cube.getSegmentById(merge2.getUuid()) != null && cube.getSegmentById(merge2.getUuid()).getStatus() == SegmentStatusEnum.NEW); } @@ -249,8 +249,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { final NavigableSet<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT); assertTrue(cubePath.size() > 1); - final List<CubeInstance> cubes = store.getAllResources(ResourceStore.CUBE_RESOURCE_ROOT, CubeInstance.class, - CubeManager.CUBE_SERIALIZER); + final List<CubeInstance> cubes = store.getAllResources(ResourceStore.CUBE_RESOURCE_ROOT, CubeInstance.class, CubeManager.CUBE_SERIALIZER); assertEquals(cubePath.size(), cubes.size()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java index 2dddcf5..17c02cc 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java @@ -20,13 +20,13 @@ package org.apache.kylin.cube; import static org.junit.Assert.assertEquals; -import java.io.IOException; - import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase; import org.apache.kylin.cube.model.CubeDesc; import org.junit.Test; +import java.io.IOException; + public class CubeSpecificConfigTest extends HotLoadKylinPropertiesTestCase { @Test public void test() { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java index 2836c31..40a458a 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java @@ -49,9 +49,7 @@ public class RowKeyAttrRuleTest { @Test public void testBadDesc() throws IOException { ValidateContext vContext = new ValidateContext(); - CubeDesc desc = JsonUtil.readValue(new FileInputStream( - LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), - CubeDesc.class); + CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); desc.getRowkey().getRowKeyColumns()[2].setColumn(""); IValidatorRule<CubeDesc> rule = new RowKeyAttrRule(); rule.validate(desc, vContext); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java b/core-cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java index 480bd22..074d973 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java @@ -36,10 +36,8 @@ public class BytesSplitterTest { bytesSplitter.split(input, input.length, (byte) 127); assertEquals(2, bytesSplitter.getBufferSize()); - assertEquals("2013-02-17", - new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length)); - assertEquals("Collectibles", - new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length)); + assertEquals("2013-02-17", new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length)); + assertEquals("Collectibles", new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length)); } @Test @@ -49,11 +47,8 @@ public class BytesSplitterTest { bytesSplitter.split(input, input.length, (byte) 127); assertEquals(3, bytesSplitter.getBufferSize()); - assertEquals("2013-02-17", - new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length)); - assertEquals("", - new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length)); - assertEquals("Collectibles", - new String(bytesSplitter.getSplitBuffers()[2].value, 0, bytesSplitter.getSplitBuffers()[2].length)); + assertEquals("2013-02-17", new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length)); + assertEquals("", new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length)); + assertEquals("Collectibles", new String(bytesSplitter.getSplitBuffers()[2].value, 0, bytesSplitter.getSplitBuffers()[2].length)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java index a738ffb..b41f23b 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java @@ -48,8 +48,7 @@ public class RowKeySplitterTest extends LocalFileMetadataTestCase { RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 11, 20); // base cuboid rowkey - byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 55, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; + byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; rowKeySplitter.split(input); assertEquals(11, rowKeySplitter.getBufferSize()); @@ -62,8 +61,7 @@ public class RowKeySplitterTest extends LocalFileMetadataTestCase { RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 11, 20); // base cuboid rowkey - byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, - 9, 9, 9, 0, 10, 5 }; + byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }; rowKeySplitter.split(input); assertEquals(10, rowKeySplitter.getBufferSize()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java index 313fdea..a69f179 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java @@ -30,8 +30,7 @@ public class CombinationTest { public int findSmallerSibling(long valueBits, long valueMask) { if ((valueBits | valueMask) != valueMask) { - throw new IllegalArgumentException( - "Dismatch " + Long.toBinaryString(valueBits) + " from " + Long.toBinaryString(valueMask)); + throw new IllegalArgumentException("Dismatch " + Long.toBinaryString(valueBits) + " from " + Long.toBinaryString(valueMask)); } int n = Long.bitCount(valueMask); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java index 0b80f3c..e2a71db 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java @@ -107,14 +107,12 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { return getCubeDescManager().getCubeDesc("ci_inner_join_cube"); } - private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds, - long[] expectChildren) { + private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds, long[] expectChildren) { Set<Long> totalSpanning = Sets.newHashSet(); for (long cuboidId : cuboidIds) { List<Long> spannings = scheduler.getSpanningCuboid(cuboidId); totalSpanning.addAll(spannings); - System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " - + toString(spannings)); + System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings)); for (long child : spannings) { assertTrue(Cuboid.isValid(cube, child)); @@ -137,20 +135,16 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { testSpanningAndGetParent(scheduler, cube, new long[] { 511 }, new long[] { 504, 447, 503, 383 }); // generate 7d System.out.println("Spanning for 7D Cuboids"); - testSpanningAndGetParent(scheduler, cube, new long[] { 504, 447, 503, 383 }, - new long[] { 440, 496, 376, 439, 487, 319, 375 }); + testSpanningAndGetParent(scheduler, cube, new long[] { 504, 447, 503, 383 }, new long[] { 440, 496, 376, 439, 487, 319, 375 }); // generate 6d System.out.println("Spanning for 6D Cuboids"); - testSpanningAndGetParent(scheduler, cube, new long[] { 440, 496, 376, 439, 487, 319, 375 }, - new long[] { 432, 480, 312, 368, 423, 455, 311, 359 }); + testSpanningAndGetParent(scheduler, cube, new long[] { 440, 496, 376, 439, 487, 319, 375 }, new long[] { 432, 480, 312, 368, 423, 455, 311, 359 }); // generate 5d System.out.println("Spanning for 5D Cuboids"); - testSpanningAndGetParent(scheduler, cube, new long[] { 432, 480, 312, 368, 423, 455, 311, 359 }, - new long[] { 416, 448, 304, 352, 391, 295, 327 }); + testSpanningAndGetParent(scheduler, cube, new long[] { 432, 480, 312, 368, 423, 455, 311, 359 }, new long[] { 416, 448, 304, 352, 391, 295, 327 }); // generate 4d System.out.println("Spanning for 4D Cuboids"); - testSpanningAndGetParent(scheduler, cube, new long[] { 416, 448, 304, 352, 391, 295, 327 }, - new long[] { 384, 288, 320, 263 }); + testSpanningAndGetParent(scheduler, cube, new long[] { 416, 448, 304, 352, 391, 295, 327 }, new long[] { 384, 288, 320, 263 }); // generate 3d System.out.println("Spanning for 3D Cuboids"); testSpanningAndGetParent(scheduler, cube, new long[] { 384, 288, 320, 263 }, new long[0]); @@ -169,12 +163,10 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { testSpanningAndGetParent(scheduler, cube, new long[] { 255 }, new long[] { 135, 251, 253, 254 }); // generate 6d System.out.println("Spanning for 6D Cuboids"); - testSpanningAndGetParent(scheduler, cube, new long[] { 135, 251, 253, 254 }, - new long[] { 131, 133, 134, 249, 250, 252 }); + testSpanningAndGetParent(scheduler, cube, new long[] { 135, 251, 253, 254 }, new long[] { 131, 133, 134, 249, 250, 252 }); // generate 5d System.out.println("Spanning for 5D Cuboids"); - testSpanningAndGetParent(scheduler, cube, new long[] { 131, 133, 134, 249, 250, 252 }, - new long[] { 129, 130, 132, 248 }); + testSpanningAndGetParent(scheduler, cube, new long[] { 131, 133, 134, 249, 250, 252 }, new long[] { 129, 130, 132, 248 }); // generate 4d System.out.println("Spanning for 4D Cuboids"); testSpanningAndGetParent(scheduler, cube, new long[] { 129, 130, 132, 248 }, new long[] { 184, 240 }); @@ -332,7 +324,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { System.out.println("build tree takes: " + (System.currentTimeMillis() - start) + "ms"); } - @Test(expected = RuntimeException.class) + @Test(expected=RuntimeException.class) public void testTooManyCombination() { File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim"); twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4))); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java index 7dab04b..0de1406 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStoreTest.java @@ -85,8 +85,7 @@ public class ConcurrentDiskStoreTest extends LocalFileMetadataTestCase { t[i] = new Thread() { public void run() { try { - IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(table.getInfo()) - .setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); + IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int i = 0; for (GTRecord r : scanner) { assertEquals(data.get(i++), r); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java index dcaea54..a64fc21 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/MemDiskStoreTest.java @@ -101,8 +101,7 @@ public class MemDiskStoreTest extends LocalFileMetadataTestCase { } builder.close(); - IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) - .setFilterPushDown(null).createGTScanRequest()); + IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int i = 0; for (GTRecord r : scanner) { assertEquals(data.get(i++), r); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java index b931f69..1d1d147 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java @@ -54,13 +54,11 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase { RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); - byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, - 9, 9, 0, 10, 5 }; + byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }; rowKeyDecoder.decode(key); List<String> values = rowKeyDecoder.getValues(); - assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, FP-GTC, 0, 15]", - values.toString()); + assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, FP-GTC, 0, 15]", values.toString()); } @Ignore @@ -70,13 +68,11 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase { RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); - byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 50, 48, 50, 49, 51, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; + byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 50, 48, 50, 49, 51, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; rowKeyDecoder.decode(key); List<String> values = rowKeyDecoder.getValues(); - assertEquals("[10000000, " + millis("2012-01-02") - + ", 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString()); + assertEquals("[10000000, " + millis("2012-01-02") + ", 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString()); } @Test @@ -104,8 +100,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase { RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); rowKeyDecoder.decode(encodedKey); List<String> values = rowKeyDecoder.getValues(); - assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, åç»æ ¼å¼, 0, 15]", - values.toString()); + assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, åç»æ ¼å¼, 0, 15]", values.toString()); } private String millis(String dateStr) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java index b0c6c96..75e2458 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java @@ -71,12 +71,10 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase { byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length); - byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, - rowKeyEncoder.getHeaderLength()); + byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength()); byte[] rest = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), encodedKey.length); assertEquals(255, Bytes.toLong(cuboidId)); - assertArrayEquals( - new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest); + assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest); } @Ignore @@ -106,16 +104,13 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase { assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length); byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN); @SuppressWarnings("unused") - byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), - 4 + rowKeyEncoder.getHeaderLength()); - byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, - rowKeyEncoder.getHeaderLength()); + byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 4 + rowKeyEncoder.getHeaderLength()); + byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength()); byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length); assertEquals(0, Bytes.toShort(shard)); // assertTrue(Bytes.toString(sellerId).startsWith("123456789")); assertEquals(511, Bytes.toLong(cuboidId)); - assertArrayEquals(new byte[] { 11, 55, -13, 49, 49, 56, 52, 56, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 22, 34, - 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest); + assertArrayEquals(new byte[] { 11, 55, -13, 49, 49, 56, 52, 56, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest); } @Ignore @@ -144,16 +139,13 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase { byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length); byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN); - byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, - rowKeyEncoder.getHeaderLength()); + byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength()); @SuppressWarnings("unused") - byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), - 18 + rowKeyEncoder.getHeaderLength()); + byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 18 + rowKeyEncoder.getHeaderLength()); byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length); assertEquals(0, Bytes.toShort(shard)); //assertTrue(Bytes.toString(sellerId).startsWith("123456789")); assertEquals(511, Bytes.toLong(cuboidId)); - assertArrayEquals(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, rest); + assertArrayEquals(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, rest); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java index 9cfccb3..0dd9b76 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java @@ -30,6 +30,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.List; +import com.google.common.collect.Lists; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; @@ -41,8 +42,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Lists; - public class DictionaryRuleTest extends LocalFileMetadataTestCase { private static KylinConfig config; @@ -75,16 +74,13 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase { @Test public void testBadDesc() throws IOException { - testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, - DictionaryDesc.create("ORDER_ID", null, "FakeBuilderClass")); - testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, - DictionaryDesc.create("ORDER_ID", null, GlobalDictionaryBuilder.class.getName())); + testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("ORDER_ID", null, "FakeBuilderClass")); + testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("ORDER_ID", null, GlobalDictionaryBuilder.class.getName())); } @Test public void testBadDesc2() throws IOException { - testDictionaryDesc(ERROR_REUSE_BUILDER_BOTH_SET, - DictionaryDesc.create("lstg_site_id", "SITE_NAME", "FakeBuilderClass")); + testDictionaryDesc(ERROR_REUSE_BUILDER_BOTH_SET, DictionaryDesc.create("lstg_site_id", "SITE_NAME", "FakeBuilderClass")); } @Test @@ -94,7 +90,8 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase { @Test public void testBadDesc4() throws IOException { - testDictionaryDesc(ERROR_TRANSITIVE_REUSE, DictionaryDesc.create("lstg_site_id", "SELLER_ID", null), + testDictionaryDesc(ERROR_TRANSITIVE_REUSE, + DictionaryDesc.create("lstg_site_id", "SELLER_ID", null), DictionaryDesc.create("price", "lstg_site_id", null)); } @@ -111,8 +108,7 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase { private void testDictionaryDesc(String expectMessage, DictionaryDesc... descs) throws IOException { DictionaryRule rule = new DictionaryRule(); - File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA - + "/cube_desc/test_kylin_cube_without_slr_left_join_desc.json"); + File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_without_slr_left_join_desc.json"); CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); List<DictionaryDesc> newDicts = Lists.newArrayList(desc.getDictionaries()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java index c9139fd..5368e16 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java @@ -25,6 +25,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.List; +import com.google.common.collect.Lists; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; @@ -35,8 +36,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Lists; - public class FunctionRuleTest extends LocalFileMetadataTestCase { private static KylinConfig config; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java index 891dc80..a0fb4a3 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java @@ -88,8 +88,7 @@ public class ProjectManagerTest extends LocalFileMetadataTestCase { System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1); - assertTrue(prjMgr.listAllRealizations("ALIEN").iterator().next().getName() - .equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT")); + assertTrue(prjMgr.listAllRealizations("ALIEN").iterator().next().getName().equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT")); assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1); prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "default", null); @@ -127,8 +126,7 @@ public class ProjectManagerTest extends LocalFileMetadataTestCase { store.deleteResource("/cube/new_cube_in_default.json"); CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); - CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, - null); + CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default")); System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
