KYLIN-1557 Enhance the check on aggregation group dimension number Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6304e947 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6304e947 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6304e947
Branch: refs/heads/1.5.x-HBase1.1.3 Commit: 6304e9474f441456d2f904fcbd6727cbb13eae66 Parents: 6fbdf87 Author: shaofengshi <shaofeng...@apache.org> Authored: Tue Apr 5 11:42:34 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Apr 5 11:43:02 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 5 ++++ .../org/apache/kylin/cube/model/CubeDesc.java | 24 ++++++++++++-------- .../org/apache/kylin/cube/CubeDescTest.java | 6 ++--- 3 files changed, 23 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6304e947/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 6a0f860..9db3081 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -379,10 +379,15 @@ abstract public class KylinConfigBase implements Serializable { return Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "8")); } + @Deprecated public int getCubeAggrGroupMaxSize() { return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12")); } + public int getCubeAggrGroupMaxCombination() { + return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.combination", "4096")); + } + public String[] getCubeDimensionCustomEncodingFactories() { return getOptionalStringArray("kylin.cube.dimension.customEncodingFactories", new String[0]); } http://git-wip-us.apache.org/repos/asf/kylin/blob/6304e947/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index b103e4f..f9ec99e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -557,20 +557,20 @@ public class CubeDesc extends RootPersistentEntity { } public void validate() { - int maxSize = config.getCubeAggrGroupMaxSize(); int index = 0; for (AggregationGroup agg : getAggregationGroups()) { if (agg.getIncludes() == null) { - logger.error("Aggregation group " + index + " includes field not set"); + logger.error("Aggregation group " + index + " 'includes' field not set"); throw new IllegalStateException("Aggregation group " + index + " includes field not set"); } if (agg.getSelectRule() == null) { - logger.error("Aggregation group " + index + " includes field not set"); + logger.error("Aggregation group " + index + " 'select_rule' field not set"); throw new IllegalStateException("Aggregation group " + index + " select rule field not set"); } + int combination = 1; Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); getDims(includeDims, agg.getIncludes()); @@ -580,10 +580,17 @@ public class CubeDesc extends RootPersistentEntity { ArrayList<Set<String>> hierarchyDimsList = Lists.newArrayList(); Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); getDims(hierarchyDimsList, hierarchyDims, agg.getSelectRule().hierarchy_dims); + for (Set<String> hierarchy : hierarchyDimsList) { + combination = combination * (hierarchy.size() + 1); + } ArrayList<Set<String>> jointDimsList = Lists.newArrayList(); Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims); + if (jointDimsList.size() > 0) { + combination = combination * (1 << jointDimsList.size()); + } + if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { logger.error("Aggregation group " + index + " Include dims not containing all the used dims"); @@ -596,13 +603,12 @@ public class CubeDesc extends RootPersistentEntity { normalDims.removeAll(hierarchyDims); normalDims.removeAll(jointDims); - int normalDimSize = normalDims.size(); - int hierarchyDimSize = hierarchyDimsList.size(); - int jointDimSize = jointDimsList.size(); + combination = combination * (1 << normalDims.size()); - if (mandatoryDims.size() + normalDimSize + hierarchyDimSize + jointDimSize > maxSize) { - logger.error("Aggregation group " + index + " has too many dimensions"); - throw new IllegalStateException("Aggregation group " + index + " has too many dimensions"); + if (combination > config.getCubeAggrGroupMaxCombination()) { + String msg = "Aggregation group " + index + " has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max.combination' to a bigger value."; + logger.error(msg); + throw new IllegalStateException(msg); } if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) { http://git-wip-us.apache.org/repos/asf/kylin/blob/6304e947/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index 141a22e..fb2e18b 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -95,14 +95,14 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testBadInit4() throws Exception { thrown.expect(IllegalStateException.class); - thrown.expectMessage("Aggregation group 0 has too many dimensions"); + thrown.expectMessage("Aggregation group 0 has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max.combination' to a bigger value."); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); try { - System.setProperty("kylin.cube.aggrgroup.max.size", "3"); + System.setProperty("kylin.cube.aggrgroup.max.combination", "8"); cubeDesc.validate(); } finally { - System.clearProperty("kylin.cube.aggrgroup.max.size"); + System.clearProperty("kylin.cube.aggrgroup.max.combination"); } }
