http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java index 2c502bb..3eba78b 100644 --- a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterTest.java @@ -97,12 +97,10 @@ public class TopNCounterTest { //@Test public void testSingleSpaceSaving() throws IOException { String dataFile = prepareTestDate(); - TopNCounterTest.SpaceSavingConsumer spaceSavingCounter = new TopNCounterTest.SpaceSavingConsumer( - TOP_K * SPACE_SAVING_ROOM); + TopNCounterTest.SpaceSavingConsumer spaceSavingCounter = new TopNCounterTest.SpaceSavingConsumer(TOP_K * SPACE_SAVING_ROOM); TopNCounterTest.HashMapConsumer accurateCounter = new TopNCounterTest.HashMapConsumer(); - for (TopNCounterTest.TestDataConsumer consumer : new TopNCounterTest.TestDataConsumer[] { spaceSavingCounter, - accurateCounter }) { + for (TopNCounterTest.TestDataConsumer consumer : new TopNCounterTest.TestDataConsumer[] { spaceSavingCounter, accurateCounter }) { feedDataToConsumer(dataFile, consumer, 0, TOTAL_RECORDS); } @@ -111,8 +109,7 @@ public class TopNCounterTest { compareResult(spaceSavingCounter, accurateCounter); } - private void compareResult(TopNCounterTest.TestDataConsumer firstConsumer, - TopNCounterTest.TestDataConsumer secondConsumer) { + private void compareResult(TopNCounterTest.TestDataConsumer firstConsumer, TopNCounterTest.TestDataConsumer secondConsumer) { List<Pair<String, Double>> topResult1 = firstConsumer.getTopN(TOP_K); outputMsg("Get topN, Space saving takes " + firstConsumer.getSpentTime() / 1000 + " seconds"); List<Pair<String, Double>> realSequence = secondConsumer.getTopN(TOP_K); @@ -126,10 +123,8 @@ public class TopNCounterTest { // if (topResult1.get(i).getFirst().equals(realSequence.get(i).getFirst()) && topResult1.get(i).getSecond().doubleValue() == realSequence.get(i).getSecond().doubleValue()) { outputMsg("Passed; key:" + topResult1.get(i).getFirst() + ", value:" + topResult1.get(i).getSecond()); } else { - outputMsg("Failed; space saving key:" + topResult1.get(i).getFirst() + ", value:" - + topResult1.get(i).getSecond()); - outputMsg("Failed; correct key:" + realSequence.get(i).getFirst() + ", value:" - + realSequence.get(i).getSecond()); + outputMsg("Failed; space saving key:" + topResult1.get(i).getFirst() + ", value:" + topResult1.get(i).getSecond()); + outputMsg("Failed; correct key:" + realSequence.get(i).getFirst() + ", value:" + realSequence.get(i).getSecond()); error++; } } @@ -172,8 +167,7 @@ public class TopNCounterTest { } - private TopNCounterTest.SpaceSavingConsumer[] singleMerge(TopNCounterTest.SpaceSavingConsumer[] consumers) - throws IOException, ClassNotFoundException { + private TopNCounterTest.SpaceSavingConsumer[] singleMerge(TopNCounterTest.SpaceSavingConsumer[] consumers) throws IOException, ClassNotFoundException { List<TopNCounterTest.SpaceSavingConsumer> list = Lists.newArrayList(); if (consumers.length == 1) return consumers; @@ -189,8 +183,7 @@ public class TopNCounterTest { } - private TopNCounterTest.SpaceSavingConsumer[] binaryMerge(TopNCounterTest.SpaceSavingConsumer[] consumers) - throws IOException, ClassNotFoundException { + private TopNCounterTest.SpaceSavingConsumer[] binaryMerge(TopNCounterTest.SpaceSavingConsumer[] consumers) throws IOException, ClassNotFoundException { List<TopNCounterTest.SpaceSavingConsumer> list = Lists.newArrayList(); if (consumers.length == 1) return consumers; @@ -206,8 +199,7 @@ public class TopNCounterTest { return binaryMerge(list.toArray(new TopNCounterTest.SpaceSavingConsumer[list.size()])); } - private void feedDataToConsumer(String dataFile, TopNCounterTest.TestDataConsumer consumer, int startLine, - int endLine) throws IOException { + private void feedDataToConsumer(String dataFile, TopNCounterTest.TestDataConsumer consumer, int startLine, int endLine) throws IOException { long startTime = System.currentTimeMillis(); BufferedReader bufferedReader = new BufferedReader(new FileReader(dataFile)); @@ -222,8 +214,7 @@ public class TopNCounterTest { } bufferedReader.close(); - outputMsg("feed data to " + consumer.getClass().getCanonicalName() + " take time (seconds): " - + (System.currentTimeMillis() - startTime) / 1000); + outputMsg("feed data to " + consumer.getClass().getCanonicalName() + " take time (seconds): " + (System.currentTimeMillis() - startTime) / 1000); } private void outputMsg(String msg) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java index 7fbef13..690e1fe 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java @@ -44,8 +44,7 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase { @Test public void testBasics() throws Exception { - BadQueryHistory history = BadQueryHistoryManager.getInstance(getTestConfig()) - .getBadQueriesForProject("default"); + BadQueryHistory history = BadQueryHistoryManager.getInstance(getTestConfig()).getBadQueriesForProject("default"); System.out.println(JsonUtil.writeValueAsIndentString(history)); NavigableSet<BadQueryEntry> entries = history.getEntries(); @@ -65,8 +64,7 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase { public void testAddEntryToProject() throws IOException { KylinConfig kylinConfig = getTestConfig(); BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig); - BadQueryHistory history = manager.addEntryToProject("sql", 1459362239992L, "adj", 100, "server", "t-0", "user", - "default"); + BadQueryHistory history = manager.addEntryToProject("sql", 1459362239992L, "adj", 100, "server", "t-0", "user", "default"); NavigableSet<BadQueryEntry> entries = history.getEntries(); assertEquals(3, entries.size()); @@ -81,8 +79,7 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase { assertEquals("t-0", newEntry.getThread()); for (int i = 0; i < kylinConfig.getBadQueryHistoryNum(); i++) { - history = manager.addEntryToProject("sql", 1459362239993L + i, "adj", 100 + i, "server", "t-0", "user", - "default"); + history = manager.addEntryToProject("sql", 1459362239993L + i, "adj", 100 + i, "server", "t-0", "user", "default"); } assertEquals(kylinConfig.getBadQueryHistoryNum(), history.getEntries().size()); } @@ -93,12 +90,10 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase { BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig); manager.addEntryToProject("sql", 1459362239000L, "adj", 100, "server", "t-0", "user", "default"); - BadQueryHistory history = manager.updateEntryToProject("sql", 1459362239000L, "adj2", 120, "server2", "t-1", - "user", "default"); + BadQueryHistory history = manager.updateEntryToProject("sql", 1459362239000L, "adj2", 120, "server2", "t-1", "user", "default"); NavigableSet<BadQueryEntry> entries = history.getEntries(); - BadQueryEntry newEntry = entries - .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user")); + BadQueryEntry newEntry = entries.floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user")); System.out.println(newEntry); assertEquals("adj2", newEntry.getAdj()); assertEquals("server2", newEntry.getServer()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java index a207975..f6d6fc3 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java @@ -57,8 +57,7 @@ public class DataModelDescTest extends LocalFileMetadataTestCase { @Test public void testNoDupColInDimAndMeasure() { - DataModelDesc model = MetadataManager.getInstance(getTestConfig()) - .getDataModelDesc("test_kylin_inner_join_model_desc"); + DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc"); String[] metrics = model.getMetrics(); TblColRef col = model.findColumn("edw.test_cal_dt.cal_dt"); assertTrue(metrics.length == 2); @@ -67,8 +66,7 @@ public class DataModelDescTest extends LocalFileMetadataTestCase { @Test public void testGetCopyOf() throws JsonProcessingException { - DataModelDesc desc = MetadataManager.getInstance(getTestConfig()) - .getDataModelDesc("test_kylin_inner_join_model_desc"); + DataModelDesc desc = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc"); DataModelDesc copyDesc = DataModelDesc.getCopyOf(desc); // uuid is different, set to equals for json comparison @@ -83,8 +81,7 @@ public class DataModelDescTest extends LocalFileMetadataTestCase { @Test public void testPartitionDescCopyOf() throws JsonProcessingException { - PartitionDesc desc = MetadataManager.getInstance(getTestConfig()) - .getDataModelDesc("test_kylin_inner_join_model_desc").partitionDesc; + PartitionDesc desc = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc").partitionDesc; PartitionDesc copyDesc = PartitionDesc.getCopyOf(desc); String descStr = JsonUtil.writeValueAsIndentString(desc); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java index ec41c6f..1b610ca 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java @@ -38,10 +38,8 @@ public class DefaultPartitionConditionBuilderTest { partitionDesc.setPartitionDateColumnRef(col); partitionDesc.setPartitionDateColumn(col.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); - String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, - DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23")); - Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", - condition); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23")); + Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition); } @Test @@ -51,8 +49,7 @@ public class DefaultPartitionConditionBuilderTest { partitionDesc.setPartitionTimeColumnRef(col); partitionDesc.setPartitionTimeColumn(col.getCanonicalName()); partitionDesc.setPartitionTimeFormat("HH"); - String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, - DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition); } @@ -67,11 +64,8 @@ public class DefaultPartitionConditionBuilderTest { partitionDesc.setPartitionTimeColumnRef(col2); partitionDesc.setPartitionTimeColumn(col2.getCanonicalName()); partitionDesc.setPartitionTimeFormat("H"); - String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, - DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); - Assert.assertEquals( - "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))", - condition); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); + Assert.assertEquals("((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))", condition); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java index a600415..d5e9de7 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java @@ -50,7 +50,7 @@ public class JoinsTreeTest extends LocalFileMetadataTestCase { MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); DataModelDesc model = mgr.getDataModelDesc("ci_left_join_model"); JoinsTree joinsTree = model.getJoinsTree(); - + Chain chain = joinsTree.tableChains.get("BUYER_COUNTRY"); assertTrue(chain.table == model.findTable("BUYER_COUNTRY")); assertTrue(chain.fkSide.table == model.findTable("BUYER_ACCOUNT")); @@ -59,7 +59,7 @@ public class JoinsTreeTest extends LocalFileMetadataTestCase { assertTrue(chain.fkSide.fkSide.fkSide.join == null); assertTrue(chain.fkSide.fkSide.fkSide.fkSide == null); } - + @Test public void testMatch() { MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java index 7d25b34..5e3ca9f 100644 --- a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java @@ -45,7 +45,7 @@ public class DataGenTest extends LocalFileMetadataTestCase { DataModelDesc model = getModel("ci_inner_join_model"); ModelDataGenerator gen = new ModelDataGenerator(model, 100); gen.outprint = true; - + gen.generate(); } @@ -54,7 +54,7 @@ public class DataGenTest extends LocalFileMetadataTestCase { DataModelDesc model = getModel("ssb"); ModelDataGenerator gen = new ModelDataGenerator(model, 100); gen.outprint = true; - + gen.generate(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java index 2ffb3bb..78cf97c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java @@ -68,8 +68,7 @@ public class StorageContext { //the limit here correspond to the limit concept in SQL //also take into consideration Statement.setMaxRows in JDBC private int getLimit() { - if (overlookOuterLimit || BackdoorToggles.getStatementMaxRows() == null - || BackdoorToggles.getStatementMaxRows() == 0) { + if (overlookOuterLimit || BackdoorToggles.getStatementMaxRows() == null || BackdoorToggles.getStatementMaxRows() == 0) { return limit; } else { return Math.min(limit, BackdoorToggles.getStatementMaxRows()); @@ -78,8 +77,7 @@ public class StorageContext { public void setLimit(int l) { if (limit != Integer.MAX_VALUE) { - logger.warn("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, - limit); + logger.warn("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit); } else { limit = l; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java b/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java index 73fde2b..7e811a4 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/adhoc/AdHocRunnerBase.java @@ -18,13 +18,13 @@ package org.apache.kylin.storage.adhoc; -import java.util.List; - import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + public abstract class AdHocRunnerBase { private static final Logger logger = LoggerFactory.getLogger(AdHocRunnerBase.class); @@ -44,6 +44,5 @@ public abstract class AdHocRunnerBase { public abstract void init(); - public abstract void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas) - throws Exception; -} + public abstract void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws Exception; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java index 84c431d..cecea85 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java @@ -72,8 +72,7 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { protected StorageContext context; - public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, - Set<TblColRef> groupByDims, // + public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupByDims, // Collection<FunctionDesc> metrics, TupleFilter havingFilter, StorageContext context) { this.context = context; @@ -100,13 +99,11 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { //replace the constant values in filter to dictionary codes Set<TblColRef> groupByPushDown = Sets.newHashSet(groupByDims); - this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), - groupByPushDown); + this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupByPushDown); this.havingFilter = havingFilter; this.gtDimensions = mapping.makeGridTableColumns(dimensions); - this.gtAggrGroups = mapping - .makeGridTableColumns(replaceDerivedColumns(groupByPushDown, cubeSegment.getCubeDesc())); + this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupByPushDown, cubeSegment.getCubeDesc())); this.gtAggrMetrics = mapping.makeGridTableColumns(metrics); this.gtAggrFuncs = mapping.makeAggrFuncs(metrics); @@ -124,8 +121,7 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { /** * Construct GTScanRangePlanner with incomplete information. For UT only. */ - public CubeScanRangePlanner(GTInfo info, Pair<ByteArray, ByteArray> gtStartAndEnd, TblColRef gtPartitionCol, - TupleFilter gtFilter) { + public CubeScanRangePlanner(GTInfo info, Pair<ByteArray, ByteArray> gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) { this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax(); this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax(); @@ -150,12 +146,9 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { List<GTScanRange> scanRanges = this.planScanRanges(); if (scanRanges != null && scanRanges.size() != 0) { scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).// - setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs) - .setFilterPushDown(gtFilter).// - setAllowStorageAggregation(context.isNeedStorageAggregation()) - .setAggCacheMemThreshold(cubeSegment.getConfig().getQueryCoprocessorMemGB()).// - setStoragePushDownLimit(context.getFinalPushDownLimit()).setHavingFilterPushDown(havingFilter) - .createGTScanRequest(); + setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).// + setAllowStorageAggregation(context.isNeedStorageAggregation()).setAggCacheMemThreshold(cubeSegment.getConfig().getQueryCoprocessorMemGB()).// + setStoragePushDownLimit(context.getFinalPushDownLimit()).setHavingFilterPushDown(havingFilter).createGTScanRequest(); } else { scanRequest = null; } @@ -210,15 +203,11 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { int beginCompare = rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond()); int endCompare = rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end); - if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) - || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) { + if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) { //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded when using dict encoding, so use <= when has equals in condition. } else { - logger.debug( - "Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", // - gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), - makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), - makeReadable(range.end)); + logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", // + gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end)); return null; } } @@ -253,8 +242,7 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { return result; } - List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet, - maxFuzzyKeys); + List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet, maxFuzzyKeys); for (Map<Integer, ByteArray> fuzzyValue : fuzzyValueCombinations) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java index 7ba1796..ee12743 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java @@ -51,11 +51,10 @@ public class CubeSegmentScanner implements IGTScanner { final GTScanRequest scanRequest; public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, // - Collection<FunctionDesc> metrics, TupleFilter originalfilter, TupleFilter havingFilter, - StorageContext context) { - + Collection<FunctionDesc> metrics, TupleFilter originalfilter, TupleFilter havingFilter, StorageContext context) { + logger.info("Init CubeSegmentScanner for segment {}", cubeSeg.getName()); - + this.cuboid = cuboid; this.cubeSeg = cubeSeg; @@ -71,20 +70,19 @@ public class CubeSegmentScanner implements IGTScanner { CubeScanRangePlanner scanRangePlanner; try { - scanRangePlanner = new CubeScanRangePlanner(cubeSeg, cuboid, filter, dimensions, groups, metrics, - havingFilter, context); + scanRangePlanner = new CubeScanRangePlanner(cubeSeg, cuboid, filter, dimensions, groups, metrics, havingFilter, context); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException(e); } - + scanRequest = scanRangePlanner.planScanRequest(); - + String gtStorage = ((GTCubeStorageQueryBase) context.getStorageQuery()).getGTStorage(); scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage, context); } - + public boolean isSegmentSkipped() { return scanner.isSegmentSkipped(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java index ccc6d33..73fc380 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java @@ -71,8 +71,7 @@ public class CubeTupleConverter implements ITupleConverter { private final int nSelectedDims; public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, // - Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, - TupleInfo returnTupleInfo) { + Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo returnTupleInfo) { this.cubeSeg = cubeSeg; this.cuboid = cuboid; this.gtColIdx = gtColIdx; @@ -110,8 +109,7 @@ public class CubeTupleConverter implements ITupleConverter { MeasureType<?> measureType = metric.getMeasureType(); if (measureType.needAdvancedTupleFilling()) { - Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap( - measureType.getColumnsNeedDictionary(metric)); + Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric)); advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap)); advMeasureIndexInGTValues.add(i); } else { @@ -122,8 +120,7 @@ public class CubeTupleConverter implements ITupleConverter { } // prepare derived columns and filler - Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc() - .getHostToDerivedInfo(cuboid.getColumns(), null); + Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboid.getColumns(), null); for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) { TblColRef[] hostCols = entry.getKey().data; for (DeriveInfo deriveInfo : entry.getValue()) { @@ -276,27 +273,22 @@ public class CubeTupleConverter implements ITupleConverter { String[] pkCols = join.getPrimaryKey(); String snapshotResPath = cubeSegment.getSnapshotResPath(tableName); if (snapshotResPath == null) - throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" - + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); + throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); try { SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotResPath); TableDesc tableDesc = metaMgr.getTableDesc(tableName); - EnhancedStringLookupTable enhancedStringLookupTable = new EnhancedStringLookupTable(tableDesc, pkCols, - snapshot); - logger.info("Time to get lookup up table for {} is {} ", join.getPKSide().getTableName(), - (System.currentTimeMillis() - ts)); + EnhancedStringLookupTable enhancedStringLookupTable = new EnhancedStringLookupTable(tableDesc, pkCols, snapshot); + logger.info("Time to get lookup up table for {} is {} ", join.getPKSide().getTableName(), (System.currentTimeMillis() - ts)); return enhancedStringLookupTable; } catch (IOException e) { - throw new IllegalStateException( - "Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); + throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); } } private static class EnhancedStringLookupTable extends LookupStringTable { - public EnhancedStringLookupTable(TableDesc tableDesc, String[] keyColumns, IReadableTable table) - throws IOException { + public EnhancedStringLookupTable(TableDesc tableDesc, String[] keyColumns, IReadableTable table) throws IOException { super(tableDesc, keyColumns, table); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 7efa76f..4ac8967 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -85,8 +85,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { continue; } - scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), - request.getMetrics(), request.getFilter(), request.getHavingFilter(), request.getContext()); + scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), request.getMetrics(), request.getFilter(), request.getHavingFilter(), request.getContext()); if (!scanner.isSegmentSkipped()) scanners.add(scanner); } @@ -94,12 +93,10 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { if (scanners.isEmpty()) return ITupleIterator.EMPTY_TUPLE_ITERATOR; - return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(), - request.getMetrics(), returnTupleInfo, request.getContext()); + return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(), request.getMetrics(), returnTupleInfo, request.getContext()); } - protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest, - TupleInfo returnTupleInfo) { + protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { context.setStorageQuery(this); //cope with queries with no aggregations @@ -145,23 +142,18 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { TupleFilter.collectColumns(filter, filterColumnD); // set limit push down - enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, - sqlDigest.aggregations, context); + enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context); // set whether to aggregate results from multiple partitions enableStreamAggregateIfBeneficial(cuboid, groupsD, context); // set query deadline context.setDeadline(cubeInstance); // push down having clause filter if possible - TupleFilter havingFilter = checkHavingCanPushDown(sqlDigest.havingFilter, groupsD, sqlDigest.aggregations, - metrics); + TupleFilter havingFilter = checkHavingCanPushDown(sqlDigest.havingFilter, groupsD, sqlDigest.aggregations, metrics); - logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", - cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), - context.isNeedStorageAggregation()); + logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation()); - return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, filterColumnD, metrics, filterD, - havingFilter, context); + return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, filterColumnD, metrics, filterD, havingFilter, context); } protected abstract String getGTStorage(); @@ -170,13 +162,11 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { return Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics); } - protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, - Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) { + protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) { return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo); } - protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, - Collection<FunctionDesc> metrics) { + protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) { for (FunctionDesc func : sqlDigest.aggregations) { if (!func.isDimensionAsMetric()) { // use the FunctionDesc from cube desc as much as possible, that has more info such as HLLC precision @@ -186,8 +176,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { for (TblColRef column : sqlDigest.allColumns) { // skip measure columns - if (sqlDigest.metricColumns.contains(column) - && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column))) { + if (sqlDigest.metricColumns.contains(column) && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column))) { continue; } @@ -236,8 +225,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { if (f instanceof CompareTupleFilter) { CompareTupleFilter compFilter = (CompareTupleFilter) f; // is COL=const ? - if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 - && compFilter.getColumn() != null) { + if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 && compFilter.getColumn() != null) { result.add(compFilter.getColumn()); } } @@ -264,8 +252,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { return resultD; } - public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, - Collection<TblColRef> singleValueD) { + public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD) { HashSet<TblColRef> temp = Sets.newHashSet(); temp.addAll(groupD); temp.addAll(singleValueD); @@ -352,9 +339,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } } - private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, - Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, - Set<TblColRef> loosenedColumnD, Collection<FunctionDesc> functionDescs, StorageContext context) { + private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, Set<TblColRef> loosenedColumnD, Collection<FunctionDesc> functionDescs, StorageContext context) { boolean possible = true; if (!TupleFilter.isEvaluableRecursively(filter)) { @@ -375,17 +360,15 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { // derived aggregation is bad, unless expanded columns are already in group by if (!groups.containsAll(derivedPostAggregation)) { possible = false; - logger.debug("Storage limit push down is impossible because derived column require post aggregation: " - + derivedPostAggregation); + logger.debug("Storage limit push down is impossible because derived column require post aggregation: " + derivedPostAggregation); } //if groupsD is clustered at "head" of the rowkey, then limit push down is possible int size = groupsD.size(); if (!groupsD.containsAll(cuboid.getColumns().subList(0, size))) { possible = false; - logger.debug( - "Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD // - + " with cuboid columns: " + cuboid.getColumns()); + logger.debug("Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD // + + " with cuboid columns: " + cuboid.getColumns()); } //if exists measures like max(cal_dt), then it's not a perfect cuboid match, cannot apply limit @@ -413,8 +396,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } if (!shardByInGroups.isEmpty()) { enabled = false; - logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: " - + shardByInGroups); + logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: " + shardByInGroups); } if (!context.isNeedStorageAggregation()) { @@ -446,8 +428,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } } - private TupleFilter checkHavingCanPushDown(TupleFilter havingFilter, Set<TblColRef> groupsD, - List<FunctionDesc> aggregations, Set<FunctionDesc> metrics) { + private TupleFilter checkHavingCanPushDown(TupleFilter havingFilter, Set<TblColRef> groupsD, List<FunctionDesc> aggregations, Set<FunctionDesc> metrics) { // must have only one segment Segments<CubeSegment> readySegs = cubeInstance.getSegments(SegmentStatusEnum.READY); if (readySegs.size() != 1) http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java index 38b457d..7793515 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java @@ -39,8 +39,7 @@ public class GTCubeStorageQueryRequest implements Serializable { private StorageContext context; public GTCubeStorageQueryRequest(Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, // - Set<TblColRef> filterCols, Set<FunctionDesc> metrics, TupleFilter filter, TupleFilter havingFilter, - StorageContext context) { + Set<TblColRef> filterCols, Set<FunctionDesc> metrics, TupleFilter filter, TupleFilter havingFilter, StorageContext context) { this.cuboid = cuboid; this.dimensions = dimensions; this.groups = groups; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java index 499d60c..474e1e0 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java @@ -18,14 +18,13 @@ package org.apache.kylin.storage.gtrecord; -import java.nio.ByteBuffer; -import java.util.NoSuchElementException; - +import com.google.common.collect.UnmodifiableIterator; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; -import com.google.common.collect.UnmodifiableIterator; +import java.nio.ByteBuffer; +import java.util.NoSuchElementException; /** * Support iterate over {@code GTRecord}s in storage partition result. http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java index 2d86bf5..8f64bd1 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java @@ -39,8 +39,7 @@ public class ScannerWorker { private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class); private IGTScanner internal = null; - public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage, - StorageContext context) { + public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage, StorageContext context) { if (scanRequest == null) { logger.info("Segment {} will be skipped", segment); internal = new EmptyGTScanner(); @@ -50,16 +49,13 @@ public class ScannerWorker { final GTInfo info = scanRequest.getInfo(); try { - IGTStorage rpc = (IGTStorage) Class.forName(gtStorage) - .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class) - .newInstance(segment, cuboid, info, context); // default behavior + IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class).newInstance(segment, cuboid, info, context); // default behavior internal = rpc.getGTScanner(scanRequest); - } catch (IOException | InstantiationException | InvocationTargetException | IllegalAccessException - | ClassNotFoundException | NoSuchMethodException e) { + } catch (IOException | InstantiationException | InvocationTargetException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException e) { throw new RuntimeException(e); } } - + public boolean isSegmentSkipped() { return internal instanceof EmptyGTScanner; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java index 7bbbc63..3bac5ec 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import com.google.common.collect.UnmodifiableIterator; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; import org.apache.kylin.gridtable.GTInfo; @@ -42,8 +43,6 @@ import org.apache.kylin.storage.StorageContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.UnmodifiableIterator; - public class SegmentCubeTupleIterator implements ITupleIterator { private static final Logger logger = LoggerFactory.getLogger(SegmentCubeTupleIterator.class); @@ -87,7 +86,8 @@ public class SegmentCubeTupleIterator implements ITupleIterator { scanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo); } - private Iterator<Object[]> getGTValuesIterator(final Iterator<GTRecord> records, final GTScanRequest scanRequest, + private Iterator<Object[]> getGTValuesIterator( + final Iterator<GTRecord> records, final GTScanRequest scanRequest, final int[] gtDimsIdx, final int[] gtMetricsIdx) { boolean hasMultiplePartitions = records instanceof SortMergedPartitionResultIterator; @@ -98,8 +98,7 @@ public class SegmentCubeTupleIterator implements ITupleIterator { return scanRequest.getInfo(); } - public void close() throws IOException { - } + public void close() throws IOException {} public Iterator<GTRecord> iterator() { return records; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java index bea2761..9d5d816 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java @@ -51,25 +51,21 @@ public class SequentialCubeTupleIterator implements ITupleIterator { private int scanCount; private int scanCountDelta; - public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, - Set<TblColRef> selectedDimensions, // + public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, Set<TblColRef> selectedDimensions, // Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context) { this.context = context; this.scanners = scanners; segmentCubeTupleIterators = Lists.newArrayList(); for (CubeSegmentScanner scanner : scanners) { - segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, - selectedMetrics, returnTupleInfo, context)); + segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context)); } if (context.mergeSortPartitionResults()) { //query with limit logger.info("Using SortedIteratorMergerWithLimit to merge segment results"); - Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators - .iterator(); - tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), - getTupleDimensionComparator(cuboid, returnTupleInfo)).getIterator(); + Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator(); + tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, returnTupleInfo)).getIterator(); } else { //normal case logger.info("Using Iterators.concat to merge segment results"); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java index 14e0d86..21e61e3 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java @@ -18,28 +18,28 @@ package org.apache.kylin.storage.gtrecord; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; +import com.google.common.collect.UnmodifiableIterator; +import org.apache.kylin.gridtable.GTInfo; +import org.apache.kylin.gridtable.GTRecord; + import java.util.Comparator; import java.util.List; import java.util.NoSuchElementException; import java.util.PriorityQueue; -import org.apache.kylin.gridtable.GTInfo; -import org.apache.kylin.gridtable.GTRecord; - -import com.google.common.collect.Iterators; -import com.google.common.collect.PeekingIterator; -import com.google.common.collect.UnmodifiableIterator; - /** * Merge-sort {@code GTRecord}s in all partitions, assume each partition contains sorted elements. */ public class SortMergedPartitionResultIterator extends UnmodifiableIterator<GTRecord> { - final GTRecord record; // reuse to avoid object creation + final GTRecord record ; // reuse to avoid object creation PriorityQueue<PeekingIterator<GTRecord>> heap; - SortMergedPartitionResultIterator(List<PartitionResultIterator> partitionResults, GTInfo info, - final Comparator<GTRecord> comparator) { + SortMergedPartitionResultIterator( + List<PartitionResultIterator> partitionResults, + GTInfo info, final Comparator<GTRecord> comparator) { this.record = new GTRecord(info); Comparator<PeekingIterator<GTRecord>> heapComparator = new Comparator<PeekingIterator<GTRecord>>() { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMerger.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMerger.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMerger.java index a75a580..d5aa9d0 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMerger.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMerger.java @@ -38,13 +38,12 @@ public class SortedIteratorMerger<E> { } public Iterator<E> getIterator() { - final PriorityQueue<PeekingImpl<E>> heap = new PriorityQueue<PeekingImpl<E>>(11, - new Comparator<PeekingImpl<E>>() { - @Override - public int compare(PeekingImpl<E> o1, PeekingImpl<E> o2) { - return comparator.compare(o1.peek(), o2.peek()); - } - }); + final PriorityQueue<PeekingImpl<E>> heap = new PriorityQueue<PeekingImpl<E>>(11, new Comparator<PeekingImpl<E>>() { + @Override + public int compare(PeekingImpl<E> o1, PeekingImpl<E> o2) { + return comparator.compare(o1.peek(), o2.peek()); + } + }); while (shardSubsets.hasNext()) { Iterator<E> iterator = shardSubsets.next(); @@ -83,8 +82,7 @@ public class SortedIteratorMerger<E> { if (poll.hasNext()) { //TODO: remove this check when validated - Preconditions.checkState(comparator.compare(current, poll.peek()) < 0, - "Not sorted! current: " + current + " Next: " + poll.peek()); + Preconditions.checkState(comparator.compare(current, poll.peek()) < 0, "Not sorted! current: " + current + " Next: " + poll.peek()); heap.offer(poll); } @@ -96,6 +94,7 @@ public class SortedIteratorMerger<E> { throw new UnsupportedOperationException(); } + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java index 3e96169..ef12ff0 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java @@ -18,10 +18,9 @@ package org.apache.kylin.storage.gtrecord; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; - +import com.google.common.base.Function; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; @@ -31,9 +30,9 @@ import org.apache.kylin.storage.StorageContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; /** * scatter the blob returned from region server to a iterable of gtrecords @@ -49,8 +48,7 @@ public class StorageResponseGTScatter implements IGTScanner { private final ImmutableBitSet groupByDims; private final boolean needSorted; // whether scanner should return sorted records - public StorageResponseGTScatter(GTScanRequest scanRequest, IPartitionStreamer partitionStreamer, - StorageContext context) { + public StorageResponseGTScatter(GTScanRequest scanRequest, IPartitionStreamer partitionStreamer, StorageContext context) { this.info = scanRequest.getInfo(); this.partitionStreamer = partitionStreamer; this.blocks = partitionStreamer.asByteArrayIterator(); @@ -72,12 +70,11 @@ public class StorageResponseGTScatter implements IGTScanner { @Override public Iterator<GTRecord> iterator() { - Iterator<PartitionResultIterator> iterators = Iterators.transform(blocks, - new Function<byte[], PartitionResultIterator>() { - public PartitionResultIterator apply(byte[] input) { - return new PartitionResultIterator(input, info, columns); - } - }); + Iterator<PartitionResultIterator> iterators = Iterators.transform(blocks, new Function<byte[], PartitionResultIterator>() { + public PartitionResultIterator apply(byte[] input) { + return new PartitionResultIterator(input, info, columns); + } + }); if (!needSorted) { logger.debug("Using Iterators.concat to pipeline partition results"); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index ba41173..1b113ee 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -106,16 +106,13 @@ public class HybridInstance extends RootPersistentEntity implements IRealization RealizationRegistry registry = RealizationRegistry.getInstance(config); List<IRealization> realizationList = Lists.newArrayList(); for (int i = 0; i < realizationEntries.size(); i++) { - IRealization realization = registry.getRealization(realizationEntries.get(i).getType(), - realizationEntries.get(i).getRealization()); + IRealization realization = registry.getRealization(realizationEntries.get(i).getType(), realizationEntries.get(i).getRealization()); if (realization == null) { - logger.error("Realization '" + realizationEntries.get(i) + " is not found, remove from Hybrid '" - + this.getName() + "'"); + logger.error("Realization '" + realizationEntries.get(i) + " is not found, remove from Hybrid '" + this.getName() + "'"); continue; } if (realization.isReady() == false) { - logger.error("Realization '" + realization.getName() + " is disabled, remove from Hybrid '" - + this.getName() + "'"); + logger.error("Realization '" + realization.getName() + " is disabled, remove from Hybrid '" + this.getName() + "'"); continue; } realizationList.add(realization); @@ -223,7 +220,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization init(); return allColumnDescs; } - + @Override public List<MeasureDesc> getMeasures() { init(); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java index 7bab3de..cf40416 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; @@ -40,13 +41,10 @@ import org.apache.kylin.metadata.realization.RealizationType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Lists; - /** */ public class HybridManager implements IRealizationProvider { - public static final Serializer<HybridInstance> HYBRID_SERIALIZER = new JsonSerializer<HybridInstance>( - HybridInstance.class); + public static final Serializer<HybridInstance> HYBRID_SERIALIZER = new JsonSerializer<HybridInstance>(HybridInstance.class); private static final Logger logger = LoggerFactory.getLogger(HybridManager.class); @@ -114,8 +112,7 @@ public class HybridManager implements IRealizationProvider { } @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) - throws IOException { + public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException { if ("hybrid".equals(entity)) { String hybridName = cacheKey; @@ -124,8 +121,7 @@ public class HybridManager implements IRealizationProvider { else reloadHybridInstance(hybridName); - for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID, - hybridName)) { + for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID, hybridName)) { broadcaster.notifyProjectSchemaUpdate(prj.getName()); } } else if ("cube".equals(entity)) { @@ -155,8 +151,7 @@ public class HybridManager implements IRealizationProvider { List<HybridInstance> result = Lists.newArrayList(); for (HybridInstance hybridInstance : hybridMap.values()) { for (RealizationEntry realizationEntry : hybridInstance.getRealizationEntries()) { - if (realizationEntry.getType() == type - && realizationEntry.getRealization().equalsIgnoreCase(realizationName)) { + if (realizationEntry.getType() == type && realizationEntry.getRealization().equalsIgnoreCase(realizationName)) { result.add(hybridInstance); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java index c16fc8a..0107434 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java @@ -46,8 +46,7 @@ public class HybridStorageQuery implements IStorageQuery { } @Override - public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, - final TupleInfo returnTupleInfo) { + public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) { List<ITupleIterator> tupleIterators = Lists.newArrayList(); for (int i = 0; i < realizations.length; i++) { if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest).capable) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java index 2410ed3..56b1106 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java @@ -163,8 +163,7 @@ public class ColumnValueRange { } private boolean between(String v, String beginValue, String endValue) { - return (beginValue == null || order.compare(beginValue, v) <= 0) - && (endValue == null || order.compare(v, endValue) <= 0); + return (beginValue == null || order.compare(beginValue, v) <= 0) && (endValue == null || order.compare(v, endValue) <= 0); } // remove invalid EQ/IN values and round start/end according to dictionary http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java index f168d30..f4150fe 100755 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java @@ -50,8 +50,7 @@ public class DerivedFilterTranslator { private static final Logger logger = LoggerFactory.getLogger(DerivedFilterTranslator.class); - public static Pair<TupleFilter, Boolean> translate(LookupStringTable lookup, DeriveInfo hostInfo, - CompareTupleFilter compf) { + public static Pair<TupleFilter, Boolean> translate(LookupStringTable lookup, DeriveInfo hostInfo, CompareTupleFilter compf) { TblColRef derivedCol = compf.getColumn(); TblColRef[] hostCols = hostInfo.columns; @@ -79,8 +78,7 @@ public class DerivedFilterTranslator { SingleColumnTuple tuple = new SingleColumnTuple(derivedCol); for (String[] row : lookup.getAllRows()) { tuple.value = row[di]; - if (compf.evaluate(tuple, - FilterCodeSystemFactory.getFilterCodeSystem(derivedCol.getColumnDesc().getType()))) { + if (compf.evaluate(tuple, FilterCodeSystemFactory.getFilterCodeSystem(derivedCol.getColumnDesc().getType()))) { collect(row, pi, satisfyingHostRecords); } } @@ -89,7 +87,8 @@ public class DerivedFilterTranslator { boolean loosened; if (satisfyingHostRecords.size() > KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) { logger.info("Deciding to loosen filter on derived filter as host candidates number {} exceeds threshold {}", // - satisfyingHostRecords.size(), KylinConfig.getInstanceFromEnv().getDerivedInThreshold()); + satisfyingHostRecords.size(), KylinConfig.getInstanceFromEnv().getDerivedInThreshold() + ); translated = buildRangeFilter(hostCols, satisfyingHostRecords); loosened = true; } else { @@ -163,8 +162,7 @@ public class DerivedFilterTranslator { return and; } - private static void findMinMax(Set<Array<String>> satisfyingHostRecords, TblColRef[] hostCols, String[] min, - String[] max) { + private static void findMinMax(Set<Array<String>> satisfyingHostRecords, TblColRef[] hostCols, String[] min, String[] max) { RowKeyColumnOrder[] orders = new RowKeyColumnOrder[hostCols.length]; for (int i = 0; i < hostCols.length; i++) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java index 37b500e..bfddb1f 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java @@ -69,9 +69,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> { private long partitionColumnStartDate = Long.MIN_VALUE; private long partitionColumnEndDate = Long.MAX_VALUE; - public HBaseKeyRange(CubeSegment cubeSeg, Cuboid cuboid, byte[] startKey, byte[] stopKey, - List<Pair<byte[], byte[]>> fuzzyKeys, List<Collection<ColumnValueRange>> flatColumnValueFilter, - long partitionColumnStartDate, long partitionColumnEndDate) { + public HBaseKeyRange(CubeSegment cubeSeg, Cuboid cuboid, byte[] startKey, byte[] stopKey, List<Pair<byte[], byte[]>> fuzzyKeys, List<Collection<ColumnValueRange>> flatColumnValueFilter, long partitionColumnStartDate, long partitionColumnEndDate) { this.cubeSeg = cubeSeg; this.cuboid = cuboid; this.startKey = startKey; @@ -83,8 +81,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> { initDebugString(); } - public HBaseKeyRange(Collection<TblColRef> dimensionColumns, Collection<ColumnValueRange> andDimensionRanges, - CubeSegment cubeSeg, CubeDesc cubeDesc) { + public HBaseKeyRange(Collection<TblColRef> dimensionColumns, Collection<ColumnValueRange> andDimensionRanges, CubeSegment cubeSeg, CubeDesc cubeDesc) { this.cubeSeg = cubeSeg; long cuboidId = this.calculateCuboidID(cubeDesc, dimensionColumns); this.cuboid = Cuboid.findById(cubeDesc, cuboidId); @@ -114,8 +111,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> { stopValues.put(column, dimRange.getEndValue()); fuzzyValues.put(column, dimRange.getEqualValues()); - TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getModel().getPartitionDesc() - .getPartitionDateColumnRef(); + TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef(); if (column.equals(partitionDateColumnRef)) { initPartitionRange(dimRange); } @@ -272,7 +268,6 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> { } public boolean hitSegment() { - return cubeSeg.getDateRangeStart() <= getPartitionColumnEndDate() - && cubeSeg.getDateRangeEnd() >= getPartitionColumnStartDate(); + return cubeSeg.getDateRangeStart() <= getPartitionColumnEndDate() && cubeSeg.getDateRangeEnd() >= getPartitionColumnStartDate(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java index 68a820d..8099527 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java @@ -36,13 +36,13 @@ import org.apache.kylin.metadata.tuple.TupleInfo; /** */ public class StorageMockUtils { - + final DataModelDesc model; - + public StorageMockUtils(DataModelDesc model) { this.model = model; } - + public TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) { TupleInfo info = new TupleInfo(); int idx = 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java index c9b36e6..e089387 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java @@ -36,8 +36,7 @@ public class EhcacheTest { @Test public void basicTest() throws InterruptedException { - System.out.println("runtime used memory: " - + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); + System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); Configuration conf = new Configuration(); conf.setMaxBytesLocalHeap("100M"); @@ -55,8 +54,7 @@ public class EhcacheTest { cacheManager.addCache(testCache); - System.out.println("runtime used memory: " - + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); + System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); byte[] blob = new byte[(1024 * 80 * 1024)];//400M Random random = new Random(); for (int i = 0; i < blob.length; i++) { @@ -73,8 +71,7 @@ public class EhcacheTest { System.out.println(testCache.get("1") == null); System.out.println(testCache.getSize()); System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes()); - System.out.println("runtime used memory: " - + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); + System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); blob = new byte[(1024 * 80 * 1024)];//400M for (int i = 0; i < blob.length; i++) { @@ -86,8 +83,7 @@ public class EhcacheTest { System.out.println(testCache.get("2") == null); System.out.println(testCache.getSize()); System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes()); - System.out.println("runtime used memory: " - + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); + System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); blob = new byte[(1024 * 80 * 1024)];//400M for (int i = 0; i < blob.length; i++) { @@ -99,8 +95,7 @@ public class EhcacheTest { System.out.println(testCache.get("3") == null); System.out.println(testCache.getSize()); System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes()); - System.out.println("runtime used memory: " - + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); + System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M"); cacheManager.shutdown(); }
