KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming joint semantic
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07537a9a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07537a9a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07537a9a Branch: refs/heads/2.x-staging Commit: 07537a9aa6c6d6ed37fb039bee5b3c2eabd534d8 Parents: af6b11a Author: honma <ho...@ebay.com> Authored: Fri Nov 27 16:47:28 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Fri Dec 4 17:08:01 2015 +0800 ---------------------------------------------------------------------- .../kylin/job/dataGen/FactTableGenerator.java | 7 +- .../persistence/RootPersistentEntity.java | 2 + .../org/apache/kylin/common/util/BasicTest.java | 2 +- .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../java/org/apache/kylin/cube/CubeSegment.java | 6 +- .../org/apache/kylin/cube/cuboid/Cuboid.java | 329 +++----- .../org/apache/kylin/cube/cuboid/CuboidCLI.java | 81 +- .../kylin/cube/cuboid/CuboidScheduler.java | 256 +++--- .../gridtable/CuboidToGridTableMapping.java | 10 +- .../cube/inmemcubing/InMemCubeBuilderUtils.java | 6 +- .../org/apache/kylin/cube/kv/RowConstants.java | 2 + .../kylin/cube/model/AggregationGroup.java | 280 +++++++ .../org/apache/kylin/cube/model/CubeDesc.java | 187 ++--- .../apache/kylin/cube/model/DimensionDesc.java | 121 +-- .../kylin/cube/model/HBaseMappingDesc.java | 2 +- .../apache/kylin/cube/model/RowKeyColDesc.java | 73 +- .../org/apache/kylin/cube/model/RowKeyDesc.java | 229 +---- .../org/apache/kylin/cube/model/SelectRule.java | 30 + .../kylin/cube/model/v2/CubeBuildTypeEnum.java | 39 + .../apache/kylin/cube/model/v2/CubeDesc.java | 826 +++++++++++++++++++ .../kylin/cube/model/v2/DimensionDesc.java | 239 ++++++ .../kylin/cube/model/v2/HBaseColumnDesc.java | 138 ++++ .../cube/model/v2/HBaseColumnFamilyDesc.java | 58 ++ .../kylin/cube/model/v2/HBaseMappingDesc.java | 96 +++ .../kylin/cube/model/v2/HierarchyDesc.java | 68 ++ .../kylin/cube/model/v2/RowKeyColDesc.java | 92 +++ .../apache/kylin/cube/model/v2/RowKeyDesc.java | 295 +++++++ .../model/validation/CubeMetadataValidator.java | 9 +- .../cube/model/validation/ValidateContext.java | 5 +- .../validation/rule/AggregationGroupRule.java | 185 +++++ .../rule/AggregationGroupSizeRule.java | 66 -- .../rule/IKylinValidationConstants.java | 6 +- .../validation/rule/MandatoryColumnRule.java | 75 -- .../model/validation/rule/RowKeyAttrRule.java | 27 +- .../cube/upgrade/V2/CubeDescUpgraderV2.java | 287 +++++++ .../cube/upgrade/V2/CubeMetadataUpgradeV2.java | 177 ++++ .../org/apache/kylin/cube/util/CubingUtils.java | 30 +- .../kylin/cube/AggregationGroupRuleTest.java | 127 +++ .../cube/AggregationGroupSizeRuleTest.java | 101 --- .../org/apache/kylin/cube/CubeManagerTest.java | 2 +- .../kylin/cube/DictionaryManagerTest.java | 4 +- .../kylin/cube/MandatoryColumnRuleTest.java | 57 -- .../apache/kylin/cube/RowKeyAttrRuleTest.java | 43 +- .../kylin/cube/cuboid/CuboidSchedulerTest.java | 155 ++-- .../apache/kylin/cube/cuboid/CuboidTest.java | 10 +- .../src/test/resources/data/TEST1_desc.json | 183 ---- .../src/test/resources/data/TEST2_desc.json | 168 ---- .../src/test/resources/data/TEST3_desc.json | 182 ---- .../apache/kylin/dict/DictionaryManager.java | 10 +- .../kylin/job/manager/ExecutableManager.java | 4 +- .../kylin/metadata/model/MeasureDesc.java | 16 +- .../kylin/engine/mr/BatchCubingJobBuilder.java | 2 +- .../mr/steps/FactDistinctColumnsMapperBase.java | 2 +- .../mr/steps/FactDistinctHiveColumnsMapper.java | 3 +- .../mr/steps/MapContextGTRecordWriter.java | 2 +- .../mr/steps/MergeCuboidFromStorageMapper.java | 2 +- .../engine/mr/steps/MergeCuboidMapper.java | 2 +- .../engine/mr/steps/MergeDictionaryStep.java | 2 +- .../kylin/engine/mr/steps/CubeSamplingTest.java | 3 +- .../apache/kylin/engine/spark/SparkCubing.java | 5 +- .../spark/cube/DefaultTupleConverter.java | 3 +- .../apache/kylin/rest/service/CubeService.java | 4 +- .../rest/controller/CubeControllerTest.java | 7 +- .../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +- .../hbase/cube/v1/CubeTupleConverter.java | 6 +- .../storage/hbase/cube/v2/CubeStorageQuery.java | 2 +- .../hbase/cube/v2/CubeTupleConverter.java | 3 +- .../kylin/storage/hbase/steps/BulkLoadJob.java | 2 +- .../storage/hbase/steps/CubeHFileMapper.java | 2 +- .../storage/hbase/steps/CubeHTableUtil.java | 2 +- .../storage/hbase/steps/HBaseCuboidWriter.java | 2 +- .../storage/hbase/steps/HBaseMROutput2.java | 4 +- .../hbase/steps/HBaseMROutput2Transition.java | 2 +- .../hbase/steps/RowValueDecoderTest.java | 4 +- 74 files changed, 3599 insertions(+), 1874 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java index 5a0fee7..c033117 100644 --- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java +++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java @@ -297,10 +297,9 @@ public class FactTableGenerator { JoinDesc jDesc = dim.getJoin(); if (jDesc == null) { // column on fact table used directly as a dimension - for (String aColumn : dim.getColumn()) { - if (!factTableCol2LookupCol.containsKey(aColumn)) - usedCols.add(aColumn); - } + String aColumn = dim.getColumn(); + if (!factTableCol2LookupCol.containsKey(aColumn)) + usedCols.add(aColumn); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java index bc72c1e..c8177e6 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java @@ -90,6 +90,8 @@ abstract public class RootPersistentEntity implements AclEntity, Serializable { public void setLastModified(long lastModified) { this.lastModified = lastModified; } + + public void updateRandomUuid() { setUuid(UUID.randomUUID().toString()); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index d753a20..7c6a57f 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -30,7 +30,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -88,6 +87,7 @@ public class BasicTest { } } + @Test public void testxx() throws InterruptedException { byte[][] data = new byte[10000000][]; http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 4592b15..4338f54 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -167,7 +167,7 @@ public class CubeManager implements IRealizationProvider { return null; DictionaryManager dictMgr = getDictionaryManager(); - DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), "true", col, factTableValueProvider); + DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(),true, col, factTableValueProvider); if (dictInfo != null) { cubeSeg.putDictResPath(col, dictInfo.getResourcePath()); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java index 62df1e9..6d899f4 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java @@ -31,11 +31,11 @@ import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc; import org.apache.kylin.dict.Dictionary; import org.apache.kylin.dict.IDictionaryAware; -import org.apache.kylin.metadata.model.IBuildable; import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; -import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.metadata.realization.IRealizationSegment; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; @@ -43,8 +43,6 @@ import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Objects; import com.google.common.collect.Maps; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.IRealizationSegment; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, IRealizationSegment { http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java index d7e7d9c..d604612 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java @@ -19,30 +19,38 @@ package org.apache.kylin.cube.cuboid; import java.util.ArrayList; -import java.util.BitSet; +import java.util.Collection; import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; +import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; +import org.apache.kylin.cube.model.AggregationGroup; +import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.DimensionDesc; -import org.apache.kylin.cube.model.HierarchyDesc; import org.apache.kylin.cube.model.RowKeyColDesc; -import org.apache.kylin.cube.model.RowKeyDesc; -import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask; -import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask; import org.apache.kylin.metadata.model.TblColRef; +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Lists; + public class Cuboid implements Comparable<Cuboid> { private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>(); + //smaller is better + public final static Comparator<Long> cuboidSelectComparator = new Comparator<Long>() { + @Override + public int compare(Long o1, Long o2) { + return ComparisonChain.start().compare(Long.bitCount(o1), Long.bitCount(o2)).compare(o1, o2).result(); + } + }; + public static Cuboid findById(CubeDesc cube, byte[] cuboidID) { return findById(cube, Bytes.toLong(cuboidID)); } @@ -64,29 +72,34 @@ public class Cuboid implements Comparable<Cuboid> { } public static boolean isValid(CubeDesc cube, long cuboidID) { - RowKeyDesc rowkey = cube.getRowkey(); + for (AggregationGroup agg : cube.getAggregationGroups()) { + if (isValid(agg, cuboidID)) { + return true; + } + } + return false; + } + + private static boolean isValid(AggregationGroup agg, long cuboidID) { if (cuboidID < 0) { throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0"); } - - if (checkBaseCuboid(rowkey, cuboidID)) { - return true; + if ((cuboidID & ~agg.getPartialCubeFullMask()) != 0) { + return false; //a cuboid's parent within agg is at most partialCubeFullMask } - if (checkMandatoryColumns(rowkey, cuboidID) == false) { - return false; - } - - if (checkAggregationGroup(rowkey, cuboidID) == false) { - return false; - } + return checkMandatoryColumns(agg, cuboidID) && checkHierarchy(agg, cuboidID) && checkJoint(agg, cuboidID); + } - if (checkHierarchy(rowkey, cuboidID) == false) { - return false; + public static List<AggregationGroup> getValidAggGroupForCuboid(CubeDesc cubeDesc, long cuboidID) { + List<AggregationGroup> ret = Lists.newArrayList(); + for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { + if (isValid(agg, cuboidID)) { + ret.add(agg); + } } - - return true; + return ret; } public static long getBaseCuboidId(CubeDesc cube) { @@ -97,168 +110,111 @@ public class Cuboid implements Comparable<Cuboid> { return findById(cube, getBaseCuboidId(cube)); } - private static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) { - // add mandantory - RowKeyDesc rowkey = cubeDesc.getRowkey(); - long mandatoryColumnMask = rowkey.getMandatoryColumnMask(); - if (cuboidID < mandatoryColumnMask) { - cuboidID = cuboidID | mandatoryColumnMask; - } - - // add hierarchy - for (DimensionDesc dimension : cubeDesc.getDimensions()) { - HierarchyDesc[] hierarchies = dimension.getHierarchy(); - boolean found = false; - long result = 0; - if (hierarchies != null && hierarchies.length > 0) { - for (int i = hierarchies.length - 1; i >= 0; i--) { - TblColRef hColumn = hierarchies[i].getColumnRef(); - Integer index = rowkey.getColumnBitIndex(hColumn); - long bit = 1L << index; - - if ((rowkey.getTailMask() & bit) > 0) - continue; // ignore levels in tail, they don't participate - - if ((bit & cuboidID) > 0) { - found = true; - } - - if (found == true) { - result = result | bit; - } - } - cuboidID = cuboidID | result; + public static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) { + List<Long> candidates = Lists.newArrayList(); + for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { + Long candidate = translateToValidCuboid(agg, cuboidID); + if (candidate != null) { + candidates.add(candidate); } } - // find the left-most aggregation group - long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask(); - long leftover; - for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) { - if ((cuboidWithoutMandatory & mask.uniqueMask) > 0) { - leftover = cuboidWithoutMandatory & ~mask.groupMask; - - if (leftover == 0) { - return cuboidID; - } - - if (leftover != 0) { - cuboidID = cuboidID | mask.leftoverMask; - return cuboidID; - } - } + if (candidates.size() == 0) { + throw new IllegalStateException("Cannot find parent for :" + cuboidID); } - // doesn't have column in aggregation groups - leftover = cuboidWithoutMandatory & rowkey.getTailMask(); - if (leftover == 0) { - // doesn't have column in tail group - if (cuboidWithoutMandatory != 0) { - return cuboidID; - } else { - // no column (except mandatory), add one column - long toAddCol = (1 << (BitSet.valueOf(new long[] { rowkey.getTailMask() }).cardinality())); - // check if the toAddCol belongs to any hierarchy - List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks(); - if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) { - for (HierarchyMask hierarchyMasks : hierarchyMaskList) { - long result = toAddCol & hierarchyMasks.fullMask; - if (result > 0) { - // replace it with the root col in hierarchy - toAddCol = hierarchyMasks.allMasks[0]; - break; - } - } - } - cuboidID = cuboidID | toAddCol; - return cuboidID; - } - } - - // has column in tail group - cuboidID = cuboidID | rowkey.getTailMask(); - return cuboidID; - + return Collections.min(candidates, cuboidSelectComparator); } - /** Breadth-First-Search - * @deprecated due to poor performance - * @param cube - * @param cuboidID - * @return - */ - private static long translateToValidCuboidDeprecated(CubeDesc cube, long cuboidID) { - if (Cuboid.isValid(cube, cuboidID)) { - return cuboidID; + private static Long translateToValidCuboid(AggregationGroup agg, long cuboidID) { + if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) { + //the partial cube might not contain all required dims + return null; } - HashSet<Long> dedupped = new HashSet<Long>(); - Queue<Long> queue = new LinkedList<Long>(); - List<Long> parents = Cuboid.getAllPossibleParents(cube, cuboidID); + // add mandantory + cuboidID = cuboidID | agg.getMandatoryColumnMask(); - // check each parent - addToQueue(queue, parents, dedupped); - while (queue.size() > 0) { - long parent = pollFromQueue(queue, dedupped); - if (Cuboid.isValid(cube, parent)) { - return parent; - } else { - addToQueue(queue, Cuboid.getAllPossibleParents(cube, parent), dedupped); + // add hierarchy + for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) { + long fullMask = hierarchyMask.fullMask; + long intersect = cuboidID & fullMask; + if (intersect != 0 && intersect != fullMask) { + long lsb = Long.lowestOneBit(intersect); + long fillMask = fullMask & ~(lsb - 1); + cuboidID |= fillMask; } } - return -1; - } - - private static List<Long> getAllPossibleParents(CubeDesc cube, long cuboidID) { - List<Long> allPossibleParents = new ArrayList<Long>(); - for (int i = 0; i < cube.getRowkey().getRowKeyColumns().length; i++) { - long mask = 1L << i; - long parentId = cuboidID | mask; - if (parentId != cuboidID) { - allPossibleParents.add(parentId); + // add joint dims + for (Long joint : agg.getJoints()) { + if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) { + cuboidID = cuboidID | joint; } } - return allPossibleParents; - } - - private static void addToQueue(Queue<Long> queue, List<Long> parents, HashSet<Long> dedupped) { - Collections.sort(parents); - for (Long p : parents) { - if (!dedupped.contains(p)) { - dedupped.add(p); - queue.offer(p); + if ((cuboidID & ~agg.getMandatoryColumnMask()) != 0) { + return cuboidID; + } else { + // no column, add one column + long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()), agg.getJoints()); + if (nonJointDims != 0) { + long nonJointNonHierarchy = removeBits(nonJointDims, Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() { + @Override + public Long apply(HierarchyMask input) { + return input.fullMask; + } + })); + if (nonJointNonHierarchy != 0) { + //there exists dim that does not belong to any joint or any hierarchy, that's perfect + return cuboidID | Long.lowestOneBit(nonJointNonHierarchy); + } else { + //choose from a hierarchy that does not intersect with any joint dim, only check level 1 + long allJointDims = agg.getJointDimsMask(); + int index = 0; + for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) { + long dim = hierarchyMask.allMasks[index]; + if ((nonJointDims & allJointDims) == 0) { + return cuboidID | dim; + } + } + } } - } - } - private static long pollFromQueue(Queue<Long> queue, HashSet<Long> dedupped) { - long element = queue.poll(); - dedupped.remove(element); - return element; + return cuboidID | Collections.min(agg.getJoints(), cuboidSelectComparator); + } } - private static boolean checkBaseCuboid(RowKeyDesc rowkey, long cuboidID) { - long baseCuboidId = rowkey.getFullMask(); - if (cuboidID > baseCuboidId) { - throw new IllegalArgumentException("Cubiod " + cuboidID + " is out of scope 0-" + baseCuboidId); + private static long removeBits(long original, Collection<Long> toRemove) { + long ret = original; + for (Long joint : toRemove) { + ret = ret & ~joint; } - return baseCuboidId == cuboidID; + return ret; } - private static boolean checkMandatoryColumns(RowKeyDesc rowkey, long cuboidID) { - long mandatoryColumnMask = rowkey.getMandatoryColumnMask(); - - // note the all-zero cuboid (except for mandatory) is not valid - if (cuboidID <= mandatoryColumnMask) + private static boolean checkMandatoryColumns(AggregationGroup agg, long cuboidID) { + long mandatoryColumnMask = agg.getMandatoryColumnMask(); + if ((cuboidID & mandatoryColumnMask) != mandatoryColumnMask) { return false; + } else { + return ((cuboidID & ~mandatoryColumnMask) != 0);//cuboid with only mandatory columns is not valid + } + } - return (cuboidID & mandatoryColumnMask) == mandatoryColumnMask; + private static boolean checkJoint(AggregationGroup agg, long cuboidID) { + for (long joint : agg.getJoints()) { + long common = cuboidID & joint; + if (common == 0 || common == joint) { + } else { + return false; + } + } + return true; } - private static boolean checkHierarchy(RowKeyDesc rowkey, long cuboidID) { - List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks(); + private static boolean checkHierarchy(AggregationGroup agg, long cuboidID) { + List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks(); // if no hierarchy defined in metadata if (hierarchyMaskList == null || hierarchyMaskList.size() == 0) { return true; @@ -279,23 +235,9 @@ public class Cuboid implements Comparable<Cuboid> { return true; } - private static boolean checkAggregationGroup(RowKeyDesc rowkey, long cuboidID) { - long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask(); - long leftover; - for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) { - if ((cuboidWithoutMandatory & mask.uniqueMask) != 0) { - leftover = cuboidWithoutMandatory & ~mask.groupMask; - return leftover == 0 || leftover == mask.leftoverMask; - } - } - - leftover = cuboidWithoutMandatory & rowkey.getTailMask(); - return leftover == 0 || leftover == rowkey.getTailMask(); - } - // ============================================================================ - private CubeDesc cube; + private CubeDesc cubeDesc; private final long inputID; private final long id; private final byte[] idBytes; @@ -305,8 +247,8 @@ public class Cuboid implements Comparable<Cuboid> { private volatile CuboidToGridTableMapping cuboidToGridTableMapping = null; // will translate the cuboidID if it is not valid - private Cuboid(CubeDesc cube, long originalID, long validID) { - this.cube = cube; + private Cuboid(CubeDesc cubeDesc, long originalID, long validID) { + this.cubeDesc = cubeDesc; this.inputID = originalID; this.id = validID; this.idBytes = Bytes.toBytes(id); @@ -316,7 +258,7 @@ public class Cuboid implements Comparable<Cuboid> { private List<TblColRef> translateIdToColumns(long cuboidID) { List<TblColRef> dimesnions = new ArrayList<TblColRef>(); - RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns(); + RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { // NOTE: the order of column in list!!! long bitmask = 1L << allColumns[i].getBitIndex(); @@ -335,23 +277,30 @@ public class Cuboid implements Comparable<Cuboid> { // higher level in hierarchy can be ignored when counting aggregation columns private long eliminateHierarchyAggregation(long id) { - List<HierarchyMask> hierarchyMaskList = cube.getRowkey().getHierarchyMasks(); - if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) { - for (HierarchyMask hierMask : hierarchyMaskList) { - long[] allMasks = hierMask.allMasks; - for (int i = allMasks.length - 1; i > 0; i--) { - long bit = allMasks[i] ^ allMasks[i - 1]; - if ((inputID & bit) != 0) { - id &= ~allMasks[i - 1]; + long finalId = id; + + for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { + long temp = id; + List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks(); + if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) { + for (HierarchyMask hierMask : hierarchyMaskList) { + long[] allMasks = hierMask.allMasks; + for (int i = allMasks.length - 1; i > 0; i--) { + long bit = allMasks[i] ^ allMasks[i - 1]; + if ((inputID & bit) != 0) { + temp &= ~allMasks[i - 1]; + if (temp < finalId) + finalId = temp; + } } } } } - return id; + return finalId; } - public CubeDesc getCube() { - return cube; + public CubeDesc getCubeDesc() { + return cubeDesc; } public List<TblColRef> getColumns() { @@ -375,10 +324,6 @@ public class Cuboid implements Comparable<Cuboid> { return inputID; } - public boolean useAncestor() { - return inputID != id; - } - public boolean requirePostAggregation() { return requirePostAggregation; } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java index 7a4b00a..ec36db6 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java @@ -19,6 +19,7 @@ package org.apache.kylin.cube.cuboid; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; import java.util.TreeSet; @@ -26,9 +27,6 @@ import java.util.TreeSet; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.RowKeyDesc; -import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask; -import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask; /** * @author yangli9 @@ -41,11 +39,11 @@ public class CuboidCLI { if ("test".equals(args[0])) { CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]); - simulateCuboidGeneration(cubeDesc); + simulateCuboidGeneration(cubeDesc, true); } } - public static int simulateCuboidGeneration(CubeDesc cube) { + public static int simulateCuboidGeneration(CubeDesc cube, boolean validate) { CuboidScheduler scheduler = new CuboidScheduler(cube); long baseCuboid = Cuboid.getBaseCuboidId(cube); @@ -64,20 +62,17 @@ public class CuboidCLI { cuboidQueue.push(sc); } } - - /** disable this due to poor performance when dimension number is big - TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube); - if (enumCuboids.equals(cuboidSet) == false) { - throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet); - } - */ - int mathCount = mathCalcCuboidCount(cube); - if (mathCount != cuboidSet.size()) { - throw new IllegalStateException("Math cuboid count " + mathCount + ", but actual cuboid count " + cuboidSet.size()); + if (validate) { + //only run this for test purpose, performance is bad when # of dims is large + TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube); + System.out.println(Arrays.toString(enumCuboids.toArray(new Long[enumCuboids.size()]))); + if (enumCuboids.equals(cuboidSet) == false) { + throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet); + } } - return mathCount; + return cuboidSet.size(); } @@ -93,7 +88,7 @@ public class CuboidCLI { } public static int[] calculateAllLevelCount(CubeDesc cube) { - int levels = cube.getRowkey().getNCuboidBuildLevels(); + int levels = cube.getBuildLevel(); int[] allLevelCounts = new int[levels + 1]; CuboidScheduler scheduler = new CuboidScheduler(cube); @@ -115,56 +110,4 @@ public class CuboidCLI { return allLevelCounts; } - - public static int mathCalcCuboidCount(CubeDesc cube) { - int result = 1; // 1 for base cuboid - - RowKeyDesc rowkey = cube.getRowkey(); - AggrGroupMask[] aggrGroupMasks = rowkey.getAggrGroupMasks(); - for (int i = 0; i < aggrGroupMasks.length; i++) { - boolean hasTail = i < aggrGroupMasks.length - 1 || rowkey.getTailMask() > 0; - result += mathCalcCuboidCount_aggrGroup(rowkey, aggrGroupMasks[i], hasTail); - } - - return result; - } - - private static int mathCalcCuboidCount_aggrGroup(RowKeyDesc rowkey, AggrGroupMask aggrGroupMask, boolean hasTail) { - long groupMask = aggrGroupMask.groupMask; - int n = mathCalcCuboidCount_combination(rowkey, groupMask); - n -= 2; // exclude group all 1 and all 0 - - long nonUniqueMask = groupMask & (~aggrGroupMask.uniqueMask); - if (nonUniqueMask > 0) { - // exclude duplicates caused by non-unique columns - // FIXME this assumes non-unique masks consolidates in ONE following group which maybe not be true - n -= mathCalcCuboidCount_combination(rowkey, nonUniqueMask) - 1; // exclude all 0 - } - - if (hasTail) { - n *= 2; // tail being 1 and 0 - n += 2; // +1 for group all 1 and tail 0; +1 for group all 0 and tail 1 - } - - return n; - } - - private static int mathCalcCuboidCount_combination(RowKeyDesc rowkey, long colMask) { - if (colMask == 0) // no column selected - return 0; - - int count = 1; - - for (HierarchyMask hierMask : rowkey.getHierarchyMasks()) { - long hierBits = colMask & hierMask.fullMask; - if (hierBits != 0) { - count *= Long.bitCount(hierBits) + 1; // +1 is for all-zero case - colMask &= ~hierBits; - } - } - - count *= Math.pow(2, Long.bitCount(colMask)); - - return count; - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java index 7596601..b868782 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java @@ -21,180 +21,198 @@ package org.apache.kylin.cube.cuboid; /** */ -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import org.apache.kylin.cube.model.AggregationGroup; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.RowKeyDesc; -import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; public class CuboidScheduler { - private final CubeDesc cubeDef; - private final int size; + private final CubeDesc cubeDesc; private final long max; private final Map<Long, List<Long>> cache; - public CuboidScheduler(CubeDesc cube) { - this.cubeDef = cube; - this.size = cube.getRowkey().getRowKeyColumns().length; + public CuboidScheduler(CubeDesc cubeDesc) { + this.cubeDesc = cubeDesc; + int size = this.cubeDesc.getRowkey().getRowKeyColumns().length; this.max = (long) Math.pow(2, size) - 1; this.cache = new ConcurrentHashMap<Long, List<Long>>(); } - public int getCuboidCount() { - return getCuboidCount(Cuboid.getBaseCuboidId(cubeDef)); - } - - private int getCuboidCount(long cuboidId) { - int r = 1; - for (Long child : getSpanningCuboid(cuboidId)) { - r += getCuboidCount(child); - } - return r; - } - - public List<Long> getSpanningCuboid(long cuboid) { - if (cuboid > max || cuboid < 0) { - throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max); - } - - List<Long> result = cache.get(cuboid); - if (result != null) { - return result; + public long getParent(long child) { + List<Long> candidates = Lists.newArrayList(); + long baseCuboidID = Cuboid.getBaseCuboidId(cubeDesc); + if (child == baseCuboidID || !Cuboid.isValid(cubeDesc, child)) { + throw new IllegalStateException(); } + for (AggregationGroup agg : Cuboid.getValidAggGroupForCuboid(cubeDesc, child)) { + boolean thisAggContributed = false; + if (agg.getPartialCubeFullMask() == child) { + // candidates.add(baseCuboidID); + // continue; + return baseCuboidID; - // smaller sibling's children - Collection<Long> allPrevOffspring = new HashSet<Long>(); - for (Long sibling : findSmallerSibling(cuboid)) { - Collection<Long> prevOffsprings = generateChildren(sibling); - allPrevOffspring.addAll(prevOffsprings); - } - - // my children is my generation excluding smaller sibling's generation - result = new ArrayList<Long>(); - for (Long offspring : generateChildren(cuboid)) { - if (!allPrevOffspring.contains(offspring)) { - result.add(offspring); } - } - cache.put(cuboid, result); - return result; - } + //+1 dim - private Collection<Long> generateChildren(long cuboid) { - Collection<Long> result = new HashSet<Long>(); + //add one normal dim (only try the lowest dim) + long normalDimsMask = (agg.getNormalDimsMask() & ~child); + if (normalDimsMask != 0) { + candidates.add(child | Long.lowestOneBit(normalDimsMask)); + thisAggContributed = true; + } - // generate zero tail cuboid -- the one with all 1 in the first - // aggregation group and all 0 for the rest bits - generateZeroTailBase(cuboid, result); + for (AggregationGroup.HierarchyMask hierarchyMask : agg.getHierarchyMasks()) { + if ((child & hierarchyMask.fullMask) == 0) { + candidates.add(child | hierarchyMask.dims[0]); + thisAggContributed = true; + } else { + for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) { + if ((child & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) { + if (i == hierarchyMask.allMasks.length - 1) { + continue;//match the full hierarchy + } + if ((agg.getJointDimsMask() & hierarchyMask.dims[i + 1]) == 0) { + if ((child & hierarchyMask.dims[i + 1]) == 0) { + //only when the hierarchy dim is not among joints + candidates.add(child | hierarchyMask.dims[i + 1]); + thisAggContributed = true; + } + } + break;//if hierarchyMask 111 is matched, won't check 110 or 100 + } + } + } + } - RowKeyDesc rowkey = cubeDef.getRowkey(); - long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask(); - for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) { - if (belongTo(cuboidWithoutMandatory, mask) == false) + if (thisAggContributed) { + //next section is going to append more than 2 dim to child + //thisAggContributed means there's already 1 dim added to child + //which can safely prune the 2+ dim candidates. continue; + } - long[] groupOneBitMasks = mask.groupOneBitMasks; - for (int i = 0; i < groupOneBitMasks.length; i++) { - long oneBit = groupOneBitMasks[i]; - if ((cuboid & oneBit) == 0) - continue; - - long child = cuboid ^ oneBit; - if (Cuboid.isValid(cubeDef, child)) { - result.add(child); + //2+ dim candidates + for (long joint : agg.getJoints()) { + if ((child & joint) == 0) { + candidates.add(child | joint); } } + } - if ((cuboidWithoutMandatory & mask.uniqueMask) > 0) - break; + if (candidates.size() == 0) { + throw new IllegalStateException(); } - return result; + return Collections.min(candidates, Cuboid.cuboidSelectComparator); } - private void generateZeroTailBase(long cuboid, Collection<Long> result) { - RowKeyDesc rowkey = cubeDef.getRowkey(); + private boolean containsNonMandatoryColumn(AggregationGroup agg, long cuboid) { + return (cuboid & ~agg.getMandatoryColumnMask()) != 0; + } - long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask(); + public Set<Long> getPotentialChildren(long parent) { - for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) { - if ((cuboidWithoutMandatory & mask.groupMask) == mask.groupMask && (cuboidWithoutMandatory & mask.leftoverMask) == mask.leftoverMask) { - long zeroTail = rowkey.getMandatoryColumnMask() | mask.groupMask; - if (zeroTail > 0 && zeroTail != cuboid) { - result.add(zeroTail); + if (parent != Cuboid.getBaseCuboid(cubeDesc).getId() && !Cuboid.isValid(cubeDesc, parent)) { + throw new IllegalStateException(); + } + + HashSet<Long> set = Sets.newHashSet(); + if (Long.bitCount(parent) == 1) { + //do not aggregate apex cuboid + return set; + } + + if (parent == Cuboid.getBaseCuboidId(cubeDesc)) { + //base cuboid is responsible for spawning each agg group's root + for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { + long partialCubeFullMask = agg.getPartialCubeFullMask(); + if (partialCubeFullMask != parent && containsNonMandatoryColumn(agg, partialCubeFullMask)) { + set.add(partialCubeFullMask); } } - if ((cuboidWithoutMandatory & mask.uniqueMask) > 0) - break; } - } - public Collection<Long> findSmallerSibling(long cuboid) { - if (!Cuboid.isValid(cubeDef, cuboid)) { - return Collections.emptyList(); - } + for (AggregationGroup agg : Cuboid.getValidAggGroupForCuboid(cubeDesc, parent)) { - RowKeyDesc rowkey = cubeDef.getRowkey(); + //normal dim section + for (long normalDimMask : agg.getNormalDims()) { + long common = parent & normalDimMask; + long temp = parent ^ normalDimMask; + if (common != 0 && containsNonMandatoryColumn(agg, temp)) { + set.add(temp); + } + } - // do combination in all related groups - long groupAllBitMask = 0; - for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) { - if ((mask.groupMask & cuboid) > 0) { - groupAllBitMask |= mask.groupMask; + for (AggregationGroup.HierarchyMask hierarchyMask : agg.getHierarchyMasks()) { + for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) { + if ((parent & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) { + if ((agg.getJointDimsMask() & hierarchyMask.dims[i]) == 0) { + if (containsNonMandatoryColumn(agg, parent ^ hierarchyMask.dims[i])) { + //only when the hierarchy dim is not among joints + set.add(parent ^ hierarchyMask.dims[i]); + } + } + break;//if hierarchyMask 111 is matched, won't check 110 or 100 + } + } } + + //joint dim section + for (long joint : agg.getJoints()) { + if ((parent & joint) == joint) { + if (containsNonMandatoryColumn(agg, parent ^ joint)) { + set.add(parent ^ joint); + } + } + } + } - long groupBitValue = cuboid & groupAllBitMask; - long leftBitValue = cuboid & ~groupAllBitMask; - long[] groupOneBits = bits(groupAllBitMask); + return set; + } - Collection<Long> siblings = new HashSet<Long>(); - combination(cuboid, siblings, groupOneBits, 0, leftBitValue, Long.bitCount(groupBitValue)); - return siblings; + public int getCuboidCount() { + return getCuboidCount(Cuboid.getBaseCuboidId(cubeDesc)); } - private long[] bits(long groupAllBitMask) { - int size = Long.bitCount(groupAllBitMask); - long[] r = new long[size]; - long l = groupAllBitMask; - int i = 0; - while (l != 0) { - long bit = Long.highestOneBit(l); - r[i++] = bit; - l ^= bit; + private int getCuboidCount(long cuboidId) { + int r = 1; + for (Long child : getSpanningCuboid(cuboidId)) { + r += getCuboidCount(child); } return r; } - private void combination(long cuboid, Collection<Long> siblings, long[] bitMasks, int offset, long bitValue, int k) { - if (k == 0) { - if (Cuboid.isValid(cubeDef, bitValue)) { - siblings.add(bitValue); - } - } else { - for (int i = offset; i < bitMasks.length; i++) { - long newBitValue = bitValue | bitMasks[i]; - if (newBitValue < cuboid) { - combination(cuboid, siblings, bitMasks, i + 1, newBitValue, k - 1); - } + public List<Long> getSpanningCuboid(long cuboid) { + if (cuboid > max || cuboid < 0) { + throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max); + } + + List<Long> result = cache.get(cuboid); + if (result != null) { + return result; + } + + result = Lists.newArrayList(); + Set<Long> potentials = getPotentialChildren(cuboid); + for (Long potential : potentials) { + if (getParent(potential) == cuboid) { + result.add(potential); } } - } - private boolean belongTo(long cuboidWithoutMandatory, AggrGroupMask mask) { - long groupBits = cuboidWithoutMandatory & mask.groupMask; - long leftoverBits = cuboidWithoutMandatory & mask.leftoverMask; - return groupBits > 0 && (leftoverBits == 0 || leftoverBits == mask.leftoverMask); + cache.put(cuboid, result); + return result; } public int getCardinality(long cuboid) { @@ -206,7 +224,7 @@ public class CuboidScheduler { } public List<Long> getAllCuboidIds() { - final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDef); + final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); List<Long> result = Lists.newArrayList(); getSubCuboidIds(baseCuboidId, result); return result; http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java index c95e932..428137a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java @@ -61,7 +61,7 @@ public class CuboidToGridTableMapping { // column blocks of metrics ArrayList<BitSet> metricsColBlocks = Lists.newArrayList(); - for (HBaseColumnFamilyDesc familyDesc : cuboid.getCube().getHbaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) { for (int i = 0; i < familyDesc.getColumns().length; i++) { metricsColBlocks.add(new BitSet()); } @@ -69,7 +69,7 @@ public class CuboidToGridTableMapping { // metrics metrics2gt = LinkedListMultimap.create(); - for (MeasureDesc measure :cuboid.getCube().getMeasures()) { + for (MeasureDesc measure :cuboid.getCubeDesc().getMeasures()) { // Count distinct & holistic count distinct are equals() but different. // Ensure the holistic version if exists is always the first. FunctionDesc func = measure.getFunction(); @@ -85,7 +85,7 @@ public class CuboidToGridTableMapping { // map to column block int cbIdx = 0; - for (HBaseColumnFamilyDesc familyDesc : cuboid.getCube().getHbaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (hbaseColDesc.containsMeasure(measure.getName())) { metricsColBlocks.get(cbIdx).set(gtColIdx); @@ -102,7 +102,7 @@ public class CuboidToGridTableMapping { } nMetrics = gtColIdx - nDimensions; - assert nMetrics == cuboid.getCube().getMeasures().size(); + assert nMetrics == cuboid.getCubeDesc().getMeasures().size(); } public int getColumnCount() { @@ -156,7 +156,7 @@ public class CuboidToGridTableMapping { public Map<Integer, Integer> getDependentMetricsMap() { Map<Integer, Integer> result = Maps.newHashMap(); - List<MeasureDesc> measures = cuboid.getCube().getMeasures(); + List<MeasureDesc> measures = cuboid.getCubeDesc().getMeasures(); for (MeasureDesc child : measures) { if (child.getDependentMeasureRef() != null) { boolean ok = false; http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java index 9d819a4..d7d3f07 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java @@ -54,11 +54,11 @@ public final class InMemCubeBuilderUtils { } public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, final int measureCount) { - BitSet bitSet = BitSet.valueOf(new long[] { cuboidId }); + int cardinality = Long.bitCount(cuboidId); BitSet dimension = new BitSet(); - dimension.set(0, bitSet.cardinality()); + dimension.set(0, cardinality); BitSet metrics = new BitSet(); - metrics.set(bitSet.cardinality(), bitSet.cardinality() + measureCount); + metrics.set(cardinality, cardinality + measureCount); return Pair.newPair(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java index 62dea02..3510915 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java @@ -20,6 +20,8 @@ package org.apache.kylin.cube.kv; public class RowConstants { + public static final int ROWKEY_COL_DEFAULT_LENGTH = 256; + // row key fixed length place holder public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9; // row key lower bound http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java new file mode 100644 index 0000000..905f8dc --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.metadata.model.TblColRef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +@SuppressWarnings("serial") +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class AggregationGroup { + public static class HierarchyMask { + public long fullMask;// 00000111 + public long[] allMasks;// 00000100,00000110,00000111 + public long[] dims;// 00000100,00000100,00000001 + } + + @JsonProperty("includes") + private String[] includes; + @JsonProperty("select_rule") + private SelectRule selectRule; + + //computed + private long partialCubeFullMask; + private long mandatoryColumnMask; + private List<HierarchyMask> hierarchyMasks; + private List<Long> joints;//each long is a group + private long jointDimsMask; + private long normalDimsMask; + private long hierarchyDimsMask; + private List<Long> normalDims;//each long is a single dim + private CubeDesc cubeDesc; + + public void init(CubeDesc cubeDesc, RowKeyDesc rowKeyDesc) { + this.cubeDesc = cubeDesc; + Map<String, TblColRef> colNameAbbr = cubeDesc.buildColumnNameAbbreviation(); + + if (this.includes == null || this.includes.length == 0 || this.selectRule == null) { + throw new IllegalStateException("AggregationGroup incomplete"); + } + + buildPartialCubeFullMask(colNameAbbr, rowKeyDesc); + buildMandatoryColumnMask(colNameAbbr, rowKeyDesc); + buildHierarchyMasks(colNameAbbr, rowKeyDesc); + buildJointColumnMask(colNameAbbr, rowKeyDesc); + buildJointDimsMask(); + buildNormalDimsMask(); + buildHierarchyDimsMask(); + + } + + private void buildPartialCubeFullMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) { + Preconditions.checkState(this.includes != null); + Preconditions.checkState(this.includes.length != 0); + + partialCubeFullMask = 0L; + for (String dim : this.includes) { + TblColRef hColumn = colNameAbbr.get(dim); + Integer index = rowKeyDesc.getColumnBitIndex(hColumn); + long bit = 1L << index; + partialCubeFullMask |= bit; + } + } + + private void buildJointColumnMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) { + joints = Lists.newArrayList(); + + if (this.selectRule.joint_dims == null || this.selectRule.joint_dims.length == 0) { + return; + } + + for (String[] joint_dims : this.selectRule.joint_dims) { + if (joint_dims == null || joint_dims.length == 0) { + continue; + } + + long joint = 0L; + for (int i = 0; i < joint_dims.length; i++) { + TblColRef hColumn = colNameAbbr.get(joint_dims[i]); + Integer index = rowKeyDesc.getColumnBitIndex(hColumn); + long bit = 1L << index; + joint |= bit; + } + + Preconditions.checkState(joint != 0); + joints.add(joint); + } + } + + private void buildMandatoryColumnMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) { + mandatoryColumnMask = 0L; + + String[] mandatory_dims = this.selectRule.mandatory_dims; + if (mandatory_dims == null || mandatory_dims.length == 0) { + return; + } + + for (String dim : mandatory_dims) { + TblColRef hColumn = colNameAbbr.get(dim); + Integer index = rowKeyDesc.getColumnBitIndex(hColumn); + mandatoryColumnMask |= 1 << index; + } + + } + + private void buildHierarchyMasks(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) { + + HierarchyMask mask = new HierarchyMask(); + this.hierarchyMasks = new ArrayList<HierarchyMask>(); + + if (this.selectRule.hierarchy_dims == null || this.selectRule.hierarchy_dims.length == 0) { + return; + } + + for (String[] hierarchy_dims : this.selectRule.hierarchy_dims) { + if (hierarchy_dims == null || hierarchy_dims.length == 0) { + continue; + } + + ArrayList<Long> allMaskList = new ArrayList<Long>(); + ArrayList<Long> dimList = new ArrayList<Long>(); + for (int i = 0; i < hierarchy_dims.length; i++) { + TblColRef hColumn = colNameAbbr.get(hierarchy_dims[i]); + Integer index = rowKeyDesc.getColumnBitIndex(hColumn); + long bit = 1L << index; + + // if ((tailMask & bit) > 0) + // continue; // ignore levels in tail, they don't participate + // // aggregation group combination anyway + + mask.fullMask |= bit; + allMaskList.add(mask.fullMask); + dimList.add(bit); + } + + Preconditions.checkState(allMaskList.size() == dimList.size()); + mask.allMasks = new long[allMaskList.size()]; + mask.dims = new long[dimList.size()]; + for (int i = 0; i < allMaskList.size(); i++) { + mask.allMasks[i] = allMaskList.get(i); + mask.dims[i] = dimList.get(i); + } + + this.hierarchyMasks.add(mask); + + } + + } + + private void buildNormalDimsMask() { + //no joint, no hierarchy, no mandatory + long leftover = partialCubeFullMask & ~mandatoryColumnMask; + leftover &= ~this.jointDimsMask; + for (HierarchyMask hierarchyMask : this.hierarchyMasks) { + leftover &= ~hierarchyMask.fullMask; + } + + this.normalDimsMask = leftover; + this.normalDims = bits(leftover); + } + + private void buildHierarchyDimsMask() { + long ret = 0; + for (HierarchyMask mask : hierarchyMasks) { + ret |= mask.fullMask; + } + this.hierarchyDimsMask = ret; + } + + private List<Long> bits(long x) { + List<Long> r = Lists.newArrayList(); + long l = x; + while (l != 0) { + long bit = Long.lowestOneBit(l); + r.add(bit); + l ^= bit; + } + return r; + } + + public void buildJointDimsMask() { + long ret = 0; + for (long x : joints) { + ret |= x; + } + this.jointDimsMask = ret; + } + + public long getMandatoryColumnMask() { + return mandatoryColumnMask; + } + + public List<HierarchyMask> getHierarchyMasks() { + return hierarchyMasks; + } + + public int getBuildLevel() { + int ret = 1;//base cuboid => partial cube root + if (this.getPartialCubeFullMask() == Cuboid.getBaseCuboidId(cubeDesc)) { + ret -= 1;//if partial cube's root is base cuboid, then one round less agg + } + + ret += getNormalDims().size(); + for (HierarchyMask hierarchyMask : this.hierarchyMasks) { + ret += hierarchyMask.allMasks.length; + } + for (Long joint : joints) { + if ((joint & this.getHierarchyDimsMask()) == 0) { + ret += 1; + } + } + + return ret; + } + + public void setIncludes(String[] includes) { + this.includes = includes; + } + + public void setSelectRule(SelectRule selectRule) { + this.selectRule = selectRule; + } + + public List<Long> getJoints() { + return joints; + } + + public long getJointDimsMask() { + return jointDimsMask; + } + + public long getNormalDimsMask() { + return normalDimsMask; + } + + public long getHierarchyDimsMask() { + return hierarchyDimsMask; + } + + public List<Long> getNormalDims() { + return normalDims; + } + + public long getPartialCubeFullMask() { + return partialCubeFullMask; + } + + public String[] getIncludes() { + return includes; + } + + public SelectRule getSelectRule() { + return selectRule; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index ef563ed..913ce3e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -32,6 +31,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import javax.annotation.Nullable; + import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -57,6 +58,8 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Function; +import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -66,7 +69,7 @@ import com.google.common.collect.Maps; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class CubeDesc extends RootPersistentEntity { - public static enum DeriveType { + public enum DeriveType { LOOKUP, PK_FK } @@ -109,6 +112,8 @@ public class CubeDesc extends RootPersistentEntity { private RowKeyDesc rowkey; @JsonProperty("hbase_mapping") private HBaseMappingDesc hbaseMapping; + @JsonProperty("aggregation_gropus") + private List<AggregationGroup> aggregationGroups; @JsonProperty("signature") private String signature; @JsonProperty("notify_list") @@ -169,16 +174,14 @@ public class CubeDesc extends RootPersistentEntity { } /** - * Find FunctionDesc by Full Expression. - * - * @return + * @return all functions from each measure. */ - public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) { + public List<FunctionDesc> listAllFunctions() { + List<FunctionDesc> functions = new ArrayList<FunctionDesc>(); for (MeasureDesc m : measures) { - if (m.getFunction().equals(manualFunc)) - return m.getFunction(); + functions.add(m.getFunction()); } - return null; + return functions; } public TblColRef findColumnRef(String table, String column) { @@ -189,14 +192,6 @@ public class CubeDesc extends RootPersistentEntity { return cols.get(column); } - public DimensionDesc findDimensionByColumn(TblColRef col) { - for (DimensionDesc dim : dimensions) { - if (ArrayUtils.contains(dim.getColumnRefs(), col)) - return dim; - } - return null; - } - public DimensionDesc findDimensionByTable(String lookupTableName) { lookupTableName = lookupTableName.toUpperCase(); for (DimensionDesc dim : dimensions) @@ -205,28 +200,6 @@ public class CubeDesc extends RootPersistentEntity { return null; } - public DimensionDesc findDimensionByName(String dimName) { - dimName = dimName.toUpperCase(); - for (DimensionDesc dim : dimensions) { - if (dimName.equals(dim.getName())) - return dim; - } - return null; - } - - /** - * Get all functions from each measure. - * - * @return - */ - public List<FunctionDesc> listAllFunctions() { - List<FunctionDesc> functions = new ArrayList<FunctionDesc>(); - for (MeasureDesc m : measures) { - functions.add(m.getFunction()); - } - return functions; - } - public boolean isDerived(TblColRef col) { return derivedToHostMap.containsKey(col); } @@ -268,14 +241,6 @@ public class CubeDesc extends RootPersistentEntity { // ============================================================================ - public HBaseMappingDesc getHBaseMapping() { - return hbaseMapping; - } - - public void setHBaseMapping(HBaseMappingDesc hbaseMapping) { - this.hbaseMapping = hbaseMapping; - } - public KylinConfig getConfig() { return config; } @@ -352,6 +317,14 @@ public class CubeDesc extends RootPersistentEntity { this.rowkey = rowkey; } + public List<AggregationGroup> getAggregationGroups() { + return aggregationGroups; + } + + public void setAggregationGroups(List<AggregationGroup> aggregationGroups) { + this.aggregationGroups = aggregationGroups; + } + public String getSignature() { return signature; } @@ -387,12 +360,23 @@ public class CubeDesc extends RootPersistentEntity { if (!name.equals(cubeDesc.name)) return false; + if (!getFactTable().equals(cubeDesc.getFactTable())) return false; return true; } + public int getBuildLevel() { + return Collections.max(Collections2.transform(aggregationGroups, new Function<AggregationGroup, Integer>() { + @Nullable + @Override + public Integer apply(AggregationGroup input) { + return input.getBuildLevel(); + } + })); + } + @Override public int hashCode() { int result = 0; @@ -410,18 +394,28 @@ public class CubeDesc extends RootPersistentEntity { if (StringUtils.isBlank(getSignature())) { return true; } - return calculateSignature().equals(getSignature()); + + String calculated = calculateSignature(); + String saved = getSignature(); + return calculated.equals(saved); } public String calculateSignature() { - MessageDigest md = null; + MessageDigest md; try { md = MessageDigest.getInstance("MD5"); StringBuilder sigString = new StringBuilder(); - sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping)); - - byte[] signature = md.digest(sigString.toString().getBytes()); - return new String(Base64.encodeBase64(signature)); + sigString.append(this.name).append("|")// + .append(JsonUtil.writeValueAsString(this.modelName)).append("|")// + .append(JsonUtil.writeValueAsString(this.dimensions)).append("|")// + .append(JsonUtil.writeValueAsString(this.measures)).append("|")// + .append(JsonUtil.writeValueAsString(this.rowkey)).append("|")// + .append(JsonUtil.writeValueAsString(this.aggregationGroups)).append("|")// + .append(JsonUtil.writeValueAsString(this.hbaseMapping)); + + byte[] signature = md.digest(sigString.toString().toLowerCase().getBytes()); + String ret = new String(Base64.encodeBase64(signature)); + return ret; } catch (NoSuchAlgorithmException | JsonProcessingException e) { throw new RuntimeException("Failed to calculate signature"); } @@ -453,11 +447,14 @@ public class CubeDesc extends RootPersistentEntity { dim.init(this, tables); } - sortDimAndMeasure(); initDimensionColumns(); initMeasureColumns(); rowkey.init(this); + for (AggregationGroup agg : this.aggregationGroups) { + agg.init(this, rowkey); + } + if (hbaseMapping != null) { hbaseMapping.init(this); } @@ -477,35 +474,32 @@ public class CubeDesc extends RootPersistentEntity { // init dimension columns ArrayList<TblColRef> dimCols = Lists.newArrayList(); - String[] colStrs = dim.getColumn(); + String colStrs = dim.getColumn(); // when column is omitted, special case - if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) { + if ((colStrs == null && dim.isDerived()) || ("{FK}".equalsIgnoreCase(colStrs))) { for (TblColRef col : join.getForeignKeyColumns()) { dimCols.add(initDimensionColRef(col)); } } // normal case else { - if (colStrs == null || colStrs.length == 0) + if (StringUtils.isEmpty(colStrs)) throw new IllegalStateException("Dimension column must not be blank " + dim); - for (String colStr : colStrs) { - dimCols.add(initDimensionColRef(dim, colStr)); - } + dimCols.add(initDimensionColRef(dim, colStrs)); - // fill back column ref in hierarchy - if (dim.isHierarchy()) { - for (int i = 0; i < dimCols.size(); i++) - dim.getHierarchy()[i].setColumnRef(dimCols.get(i)); - } + // // fill back column ref in hierarchy + // if (dim.isHierarchy()) { + // for (int i = 0; i < dimCols.size(); i++) + // dim.getHierarchy()[i].setColumnRef(dimCols.get(i)); + // } } - TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]); + TblColRef[] dimColArray = dimCols.toArray(new TblColRef[dimCols.size()]); dim.setColumnRefs(dimColArray); // init derived columns - TblColRef[] hostCols = dimColArray; if (dim.isDerived()) { String[] derived = dim.getDerived(); String[][] split = splitDerivedColumnAndExtra(derived); @@ -515,7 +509,7 @@ public class CubeDesc extends RootPersistentEntity { for (int i = 0; i < derivedNames.length; i++) { derivedCols[i] = initDimensionColRef(dim, derivedNames[i]); } - initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra); + initDerivedMap(dimColArray, DeriveType.LOOKUP, dim, derivedCols, derivedExtra); } // PK-FK derive the other side @@ -526,10 +520,10 @@ public class CubeDesc extends RootPersistentEntity { allColumns.addAll(Arrays.asList(fk)); allColumns.addAll(Arrays.asList(pk)); for (int i = 0; i < fk.length; i++) { - int find = ArrayUtils.indexOf(hostCols, fk[i]); + int find = ArrayUtils.indexOf(dimColArray, fk[i]); if (find >= 0) { TblColRef derivedCol = initDimensionColRef(pk[i]); - initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol); + initDerivedMap(dimColArray[find], DeriveType.PK_FK, dim, derivedCol); } } /** disable this code as we don't need fk be derived from pk @@ -666,7 +660,7 @@ public class CubeDesc extends RootPersistentEntity { for (int i = 0; i < measures.size(); i++) measureIndexLookup.put(measures.get(i).getName(), i); - for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { String[] colMeasureRefs = c.getMeasureRefs(); MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length]; @@ -682,53 +676,6 @@ public class CubeDesc extends RootPersistentEntity { } } - private void sortDimAndMeasure() { - sortDimensionsByID(); - sortMeasuresByID(); - for (DimensionDesc dim : dimensions) { - sortHierarchiesByLevel(dim.getHierarchy()); - } - } - - private void sortDimensionsByID() { - Collections.sort(dimensions, new Comparator<DimensionDesc>() { - @Override - public int compare(DimensionDesc d1, DimensionDesc d2) { - Integer id1 = d1.getId(); - Integer id2 = d2.getId(); - return id1.compareTo(id2); - } - }); - } - - private void sortMeasuresByID() { - if (measures == null) { - measures = Lists.newArrayList(); - } - - Collections.sort(measures, new Comparator<MeasureDesc>() { - @Override - public int compare(MeasureDesc m1, MeasureDesc m2) { - Integer id1 = m1.getId(); - Integer id2 = m2.getId(); - return id1.compareTo(id2); - } - }); - } - - private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) { - if (hierarchies != null) { - Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() { - @Override - public int compare(HierarchyDesc h1, HierarchyDesc h2) { - Integer level1 = Integer.parseInt(h1.getLevel()); - Integer level2 = Integer.parseInt(h2.getLevel()); - return level1.compareTo(level2); - } - }); - } - } - public boolean hasHolisticCountDistinctMeasures() { for (MeasureDesc measure : measures) { if (measure.getFunction().isHolisticCountDistinct()) { @@ -795,7 +742,7 @@ public class CubeDesc extends RootPersistentEntity { return storageType; } - void setStorageType(int storageType) { + public void setStorageType(int storageType) { this.storageType = storageType; } @@ -803,7 +750,7 @@ public class CubeDesc extends RootPersistentEntity { return engineType; } - void setEngineType(int engineType) { + public void setEngineType(int engineType) { this.engineType = engineType; } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java index e75a123..bccae58 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java @@ -18,11 +18,10 @@ package org.apache.kylin.cube.model; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.Map; +import org.apache.commons.lang.NotImplementedException; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.LookupDesc; @@ -32,34 +31,27 @@ import org.apache.kylin.metadata.model.TblColRef; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class DimensionDesc { - @JsonProperty("id") - private int id; - @JsonProperty("name") private String name; - - @JsonProperty("hierarchy") - private boolean isHierarchy; @JsonProperty("table") private String table; @JsonProperty("column") - private String[] column; + private String column; @JsonProperty("derived") private String[] derived; private TableDesc tableDesc; private JoinDesc join; - private HierarchyDesc[] hierarchy; // computed private TblColRef[] columnRefs; - private TblColRef[] derivedColRefs; public void init(CubeDesc cubeDesc, Map<String, TableDesc> tables) { if (name != null) @@ -80,29 +72,30 @@ public class DimensionDesc { } } - if (isHierarchy && this.column.length > 0) { - List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3); - for (int i = 0, n = this.column.length; i < n; i++) { - String aColumn = this.column[i]; - HierarchyDesc aHierarchy = new HierarchyDesc(); - aHierarchy.setLevel(String.valueOf(i + 1)); - aHierarchy.setColumn(aColumn); - hierarchyList.add(aHierarchy); - } - - this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]); - } + // if (isHierarchy && this.column.length > 0) { + // List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3); + // for (int i = 0, n = this.column.length; i < n; i++) { + // String aColumn = this.column[i]; + // HierarchyDesc aHierarchy = new HierarchyDesc(); + // aHierarchy.setLevel(String.valueOf(i + 1)); + // aHierarchy.setColumn(aColumn); + // hierarchyList.add(aHierarchy); + // } + // + // this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]); + // } + // + // if (hierarchy != null && hierarchy.length == 0) + // hierarchy = null; + + // if (hierarchy != null) { + // for (HierarchyDesc h : hierarchy) + // h.setColumn(h.getColumn().toUpperCase()); + // } - if (hierarchy != null && hierarchy.length == 0) - hierarchy = null; if (derived != null && derived.length == 0) derived = null; - if (hierarchy != null) { - for (HierarchyDesc h : hierarchy) - h.setColumn(h.getColumn().toUpperCase()); - } - if (derived != null) { StringUtil.toUpperCaseArray(derived, derived); } @@ -112,29 +105,10 @@ public class DimensionDesc { } } - public boolean isHierarchyColumn(TblColRef col) { - if (hierarchy == null) - return false; - - for (HierarchyDesc hier : hierarchy) { - if (hier.getColumnRef().equals(col)) - return true; - } - return false; - } - public boolean isDerived() { return derived != null; } - public boolean isHierarchy() { - return isHierarchy; - } - - public void setHierarchy(boolean isHierarchy) { - this.isHierarchy = isHierarchy; - } - public String getTable() { return table; } @@ -143,14 +117,6 @@ public class DimensionDesc { this.table = table; } - public int getId() { - return id; - } - - public void setId(int id) { - this.id = id; - } - public JoinDesc getJoin() { return join; } @@ -171,22 +137,14 @@ public class DimensionDesc { this.columnRefs = colRefs; } - public String[] getColumn() { + public String getColumn() { return this.column; } - public void setColumn(String[] column) { + public void setColumn(String column) { this.column = column; } - public HierarchyDesc[] getHierarchy() { - return hierarchy; - } - - public void setHierarchy(HierarchyDesc[] hierarchy) { - this.hierarchy = hierarchy; - } - public String[] getDerived() { return derived; } @@ -195,45 +153,22 @@ public class DimensionDesc { this.derived = derived; } - public TblColRef[] getDerivedColRefs() { - return derivedColRefs; - } - - public void setDerivedColRefs(TblColRef[] derivedColRefs) { - this.derivedColRefs = derivedColRefs; - } - public TableDesc getTableDesc() { return this.tableDesc; } @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - DimensionDesc that = (DimensionDesc) o; - - if (id != that.id) - return false; - if (!name.equals(that.name)) - return false; - - return true; + throw new NotImplementedException(); } @Override public int hashCode() { - int result = id; - result = 31 * result + name.hashCode(); - return result; + throw new NotImplementedException(); } @Override public String toString() { - return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]"; + return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column).add("derived", Arrays.toString(derived)).add("join", join).toString(); } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java index e390bd6..2ef1e17 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java @@ -43,7 +43,7 @@ public class HBaseMappingDesc { public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); - HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping(); + HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; }
