KYLIN-1749 Allow mandatory only cuboid
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/57295d62 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/57295d62 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/57295d62 Branch: refs/heads/1.5.x-CDH5.7 Commit: 57295d624f6e93ee6f5312db7d6e056d9bb3b256 Parents: 04318ed Author: Yang Li <liy...@apache.org> Authored: Tue May 31 08:12:47 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Tue May 31 08:35:04 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../java/org/apache/kylin/cube/cuboid/Cuboid.java | 11 ++++++----- .../apache/kylin/cube/cuboid/CuboidScheduler.java | 12 ++++-------- .../apache/kylin/cube/model/AggregationGroup.java | 7 ++++++- .../org/apache/kylin/cube/cuboid/CuboidTest.java | 16 ++++++++++++++++ .../test_case_data/localmeta/cube_desc/ssb.json | 7 ++++--- 6 files changed, 40 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/57295d62/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b55c93c..15f10a9 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -391,6 +391,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.combination", "4096")); } + public boolean getCubeAggrGroupIsMandatoryOnlyValid() { + return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.isMandatoryOnlyValid", "false")); + } + public String[] getCubeDimensionCustomEncodingFactories() { return getOptionalStringArray("kylin.cube.dimension.customEncodingFactories", new String[0]); } http://git-wip-us.apache.org/repos/asf/kylin/blob/57295d62/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java index e8a75d7..a00fc68 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java @@ -102,9 +102,9 @@ public class Cuboid implements Comparable<Cuboid> { return false; } - private static boolean isValid(AggregationGroup agg, long cuboidID) { - if (cuboidID < 0) { - throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0"); + static boolean isValid(AggregationGroup agg, long cuboidID) { + if (cuboidID <= 0) { + return false; //cuboid must be greater than 0 } if ((cuboidID & ~agg.getPartialCubeFullMask()) != 0) { return false; //a cuboid's parent within agg is at most partialCubeFullMask @@ -187,7 +187,7 @@ public class Cuboid implements Comparable<Cuboid> { } } - if ((cuboidID & ~agg.getMandatoryColumnMask()) != 0) { + if (isValid(agg, cuboidID)) { return cuboidID; } else { // no column, add one column @@ -231,7 +231,8 @@ public class Cuboid implements Comparable<Cuboid> { if ((cuboidID & mandatoryColumnMask) != mandatoryColumnMask) { return false; } else { - return ((cuboidID & ~mandatoryColumnMask) != 0);//cuboid with only mandatory columns is not valid + //cuboid with only mandatory columns maybe valid + return agg.isMandatoryOnlyValid() || (cuboidID & ~mandatoryColumnMask) != 0; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/57295d62/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java index b868782..bd6a37a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java @@ -116,10 +116,6 @@ public class CuboidScheduler { return Collections.min(candidates, Cuboid.cuboidSelectComparator); } - private boolean containsNonMandatoryColumn(AggregationGroup agg, long cuboid) { - return (cuboid & ~agg.getMandatoryColumnMask()) != 0; - } - public Set<Long> getPotentialChildren(long parent) { if (parent != Cuboid.getBaseCuboid(cubeDesc).getId() && !Cuboid.isValid(cubeDesc, parent)) { @@ -136,7 +132,7 @@ public class CuboidScheduler { //base cuboid is responsible for spawning each agg group's root for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { long partialCubeFullMask = agg.getPartialCubeFullMask(); - if (partialCubeFullMask != parent && containsNonMandatoryColumn(agg, partialCubeFullMask)) { + if (partialCubeFullMask != parent && Cuboid.isValid(agg, partialCubeFullMask)) { set.add(partialCubeFullMask); } } @@ -148,7 +144,7 @@ public class CuboidScheduler { for (long normalDimMask : agg.getNormalDims()) { long common = parent & normalDimMask; long temp = parent ^ normalDimMask; - if (common != 0 && containsNonMandatoryColumn(agg, temp)) { + if (common != 0 && Cuboid.isValid(agg, temp)) { set.add(temp); } } @@ -157,7 +153,7 @@ public class CuboidScheduler { for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) { if ((parent & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) { if ((agg.getJointDimsMask() & hierarchyMask.dims[i]) == 0) { - if (containsNonMandatoryColumn(agg, parent ^ hierarchyMask.dims[i])) { + if (Cuboid.isValid(agg, parent ^ hierarchyMask.dims[i])) { //only when the hierarchy dim is not among joints set.add(parent ^ hierarchyMask.dims[i]); } @@ -170,7 +166,7 @@ public class CuboidScheduler { //joint dim section for (long joint : agg.getJoints()) { if ((parent & joint) == joint) { - if (containsNonMandatoryColumn(agg, parent ^ joint)) { + if (Cuboid.isValid(agg, parent ^ joint)) { set.add(parent ^ joint); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/57295d62/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 35f85b0..54adb6c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -31,7 +31,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -@SuppressWarnings("serial") @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class AggregationGroup { public static class HierarchyMask { @@ -55,9 +54,11 @@ public class AggregationGroup { private long hierarchyDimsMask; private List<Long> normalDims;//each long is a single dim private CubeDesc cubeDesc; + private boolean isMandatoryOnlyValid; public void init(CubeDesc cubeDesc, RowKeyDesc rowKeyDesc) { this.cubeDesc = cubeDesc; + this.isMandatoryOnlyValid = cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid(); Map<String, TblColRef> colNameAbbr = cubeDesc.buildColumnNameAbbreviation(); if (this.includes == null || this.includes.length == 0 || this.selectRule == null) { @@ -276,4 +277,8 @@ public class AggregationGroup { public SelectRule getSelectRule() { return selectRule; } + + public boolean isMandatoryOnlyValid() { + return isMandatoryOnlyValid; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/57295d62/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java index be77855..1d06d3e 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java @@ -53,6 +53,11 @@ public class CuboidTest extends LocalFileMetadataTestCase { return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); } + + private CubeDesc getSSBCubeDesc() { + return getCubeDescManager().getCubeDesc("ssb"); + } + @Before public void setUp() throws Exception { this.createTestMetadata(); @@ -141,6 +146,17 @@ public class CuboidTest extends LocalFileMetadataTestCase { } @Test + public void testIsValid3() { + CubeDesc cube = getSSBCubeDesc(); + + assertEquals(false, Cuboid.isValid(cube, toLong("10000000000"))); + + // the 4th is mandatory and isMandatoryOnlyValid is true + assertEquals(true, Cuboid.isValid(cube, toLong("10000001000"))); + assertEquals(true, Cuboid.isValid(cube, toLong("00000001000"))); + } + + @Test public void testFindCuboidByIdWithSingleAggrGroup2() { CubeDesc cube = getTestKylinCubeWithSeller(); Cuboid cuboid; http://git-wip-us.apache.org/repos/asf/kylin/blob/57295d62/examples/test_case_data/localmeta/cube_desc/ssb.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json index 95a42d3..d3ea10b 100644 --- a/examples/test_case_data/localmeta/cube_desc/ssb.json +++ b/examples/test_case_data/localmeta/cube_desc/ssb.json @@ -159,8 +159,8 @@ "aggregation_groups" : [ { "includes" : [ "LO_PARTKEY", "C_CITY", "C_REGION", "C_NATION", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ], "select_rule" : { - "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ], - "mandatory_dims" : [ ], + "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ], + "mandatory_dims" : [ "D_YEAR" ], "joint_dims" : [ ] } } ], @@ -173,6 +173,7 @@ "engine_type" : 2, "storage_type" : 2, "override_kylin_properties" : { - "kylin.hbase.default.compression.codec" : "lz4" + "kylin.hbase.default.compression.codec" : "lz4", + "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true" } } \ No newline at end of file
