Repository: kylin Updated Branches: refs/heads/1.5.1-hbase1.1.3-test e46c3673e -> 30d60e228
KYLIN-1556 remove the left hybrid and refine candidate priority for IT Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30d60e22 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30d60e22 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30d60e22 Branch: refs/heads/1.5.1-hbase1.1.3-test Commit: 30d60e2280969ef9bb58801cc2d2132cbf68b778 Parents: e46c367 Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Apr 13 11:09:55 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Thu Apr 21 17:35:42 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/cube/CubeCapabilityChecker.java | 2 +- .../kylin/metadata/model/FunctionDesc.java | 12 +++++++++++- .../metadata/realization/CapabilityResult.java | 2 +- .../hybrid/test_kylin_hybrid_left_join.json | 14 -------------- .../localmeta/project/default.json | 5 ----- .../apache/kylin/query/ITCombinationTest.java | 12 ++++++++++++ .../org/apache/kylin/query/ITIIQueryTest.java | 8 +------- .../org/apache/kylin/query/ITKylinQueryTest.java | 11 +++++++++++ .../apache/kylin/query/routing/Candidate.java | 19 ++++++++++++------- 9 files changed, 49 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index e21dc2b..bd45d07 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -177,7 +177,7 @@ public class CubeCapabilityChecker { // calcite can do aggregation from columns on-the-fly List<TblColRef> neededCols = functionDesc.getParameter().getColRefs(); - if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols)) { + if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) { result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc)); it.remove(); continue; http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index 9e3effb..f3a81d6 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Set; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.MeasureTypeFactory; @@ -31,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; /** */ @@ -41,6 +43,14 @@ public class FunctionDesc { public static final String FUNC_MIN = "MIN"; public static final String FUNC_MAX = "MAX"; public static final String FUNC_COUNT = "COUNT"; + public static final Set<String> BUILT_IN_AGGREGATIONS = Sets.newHashSet(); + + static { + BUILT_IN_AGGREGATIONS.add(FUNC_COUNT); + BUILT_IN_AGGREGATIONS.add(FUNC_MAX); + BUILT_IN_AGGREGATIONS.add(FUNC_MIN); + BUILT_IN_AGGREGATIONS.add(FUNC_SUM); + } public static final String PARAMETER_TYPE_CONSTANT = "constant"; public static final String PARAMETER_TYPE_COLUMN = "column"; @@ -67,7 +77,7 @@ public class FunctionDesc { ArrayList<TblColRef> colRefs = Lists.newArrayList(); for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) { if (p.isColumnType()) { - ColumnDesc sourceColumn = findColumn(factTable,lookupTables,p.getValue()); + ColumnDesc sourceColumn = findColumn(factTable, lookupTables, p.getValue()); TblColRef colRef = new TblColRef(sourceColumn); colRefs.add(colRef); } http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java index 3b55728..2241e6a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java @@ -53,7 +53,7 @@ public class CapabilityResult { @Override public double suggestCostMultiplier() { - return 1.0; + return 1; } public FunctionDesc getMeasureFunction() { http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json deleted file mode 100644 index 3a18f87..0000000 --- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c", - - "name": "test_kylin_hybrid_left_join", - "realizations": [ - { - "type": "CUBE", - "realization": "test_kylin_cube_with_slr_left_join_empty" - } - ], - "cost": 100, - "last_modified": 1420016227424, - "create_time": null -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/examples/test_case_data/localmeta/project/default.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index 6359ff5..41ac6da 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -39,11 +39,6 @@ "realization": "test_streaming_table_cube" }, { - "name": "test_kylin_hybrid_left_join", - "type": "HYBRID", - "realization": "test_kylin_hybrid_left_join" - }, - { "name": "test_kylin_hybrid_inner_join", "type": "HYBRID", "realization": "test_kylin_hybrid_inner_join" http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java index 1845587..4191462 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java @@ -21,7 +21,10 @@ package org.apache.kylin.query; import java.sql.SQLException; import java.util.Arrays; import java.util.Collection; +import java.util.Map; +import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.query.routing.Candidate; import org.apache.kylin.storage.hbase.HBaseStorage; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; import org.junit.AfterClass; @@ -29,6 +32,8 @@ import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import com.google.common.collect.Maps; + /** */ @RunWith(Parameterized.class) @@ -36,6 +41,12 @@ public class ITCombinationTest extends ITKylinQueryTest { @BeforeClass public static void setUp() throws SQLException { + Map<RealizationType, Integer> priorities = Maps.newHashMap(); + priorities.put(RealizationType.INVERTED_INDEX, 2); + priorities.put(RealizationType.HYBRID, 0); + priorities.put(RealizationType.CUBE, 0); + Candidate.setPriorities(priorities); + printInfo("setUp in ITCombinationTest"); } @@ -44,6 +55,7 @@ public class ITCombinationTest extends ITKylinQueryTest { printInfo("tearDown in ITCombinationTest"); clean(); HBaseStorage.overwriteStorageQuery = null; + Candidate.restorePriorities(); } /** http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java index 67033dd..3f470cb 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java @@ -46,18 +46,12 @@ public class ITIIQueryTest extends ITKylinQueryTest { priorities.put(RealizationType.HYBRID, 1); priorities.put(RealizationType.CUBE, 2); Candidate.setPriorities(priorities); - } @AfterClass public static void tearDown() throws Exception { ITKylinQueryTest.tearDown();//invoke super class - - Map<RealizationType, Integer> priorities = Maps.newHashMap(); - priorities.put(RealizationType.INVERTED_INDEX, 1); - priorities.put(RealizationType.CUBE, 0); - priorities.put(RealizationType.HYBRID, 0); - Candidate.setPriorities(priorities); + Candidate.restorePriorities(); } @Parameterized.Parameters http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 09fcf4b..7de0484 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -23,14 +23,18 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.sql.DriverManager; import java.util.List; +import java.util.Map; import java.util.Properties; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.query.enumerator.OLAPQuery; import org.apache.kylin.query.relnode.OLAPContext; +import org.apache.kylin.query.routing.Candidate; import org.apache.kylin.query.schema.OLAPSchemaFactory; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; import org.dbunit.database.DatabaseConnection; @@ -45,6 +49,12 @@ public class ITKylinQueryTest extends KylinTestBase { @BeforeClass public static void setUp() throws Exception { + Map<RealizationType, Integer> priorities = Maps.newHashMap(); + priorities.put(RealizationType.INVERTED_INDEX, 2); + priorities.put(RealizationType.HYBRID, 0); + priorities.put(RealizationType.CUBE, 0); + Candidate.setPriorities(priorities); + printInfo("setUp in KylinQueryTest"); joinType = "left"; @@ -53,6 +63,7 @@ public class ITKylinQueryTest extends KylinTestBase { @AfterClass public static void tearDown() throws Exception { + Candidate.restorePriorities(); printInfo("tearDown in KylinQueryTest"); clean(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/query/src/main/java/org/apache/kylin/query/routing/Candidate.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java index 9be8f5e..28c60a8 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java +++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java @@ -29,18 +29,23 @@ import com.google.common.collect.Maps; public class Candidate implements Comparable<Candidate> { - static final Map<RealizationType, Integer> PRIORITIES = Maps.newHashMap(); + static Map<RealizationType, Integer> DEFAULT_PRIORITIES = Maps.newHashMap(); + static Map<RealizationType, Integer> PRIORITIES = DEFAULT_PRIORITIES; static { - PRIORITIES.put(RealizationType.HYBRID, 0); - PRIORITIES.put(RealizationType.CUBE, 1); - PRIORITIES.put(RealizationType.INVERTED_INDEX, 2); + DEFAULT_PRIORITIES.put(RealizationType.HYBRID, 0); + DEFAULT_PRIORITIES.put(RealizationType.CUBE, 1); + DEFAULT_PRIORITIES.put(RealizationType.INVERTED_INDEX, 2); } /** for test only */ public static void setPriorities(Map<RealizationType, Integer> priorities) { - PRIORITIES.clear(); - PRIORITIES.putAll(priorities); + PRIORITIES = priorities; + } + + /** for test only */ + public static void restorePriorities() { + PRIORITIES = DEFAULT_PRIORITIES; } // ============================================================================ @@ -87,7 +92,7 @@ public class Candidate implements Comparable<Candidate> { if (comp != 0) { return comp; } - + return 0; }
