APACHE-KYLIN-2822: backend support for sunburst chart to show cuboid tree
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0b12df77 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0b12df77 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0b12df77 Branch: refs/heads/yaho-cube-planner Commit: 0b12df776348c7863bea6363ceacfe7380cb4e15 Parents: 7279427 Author: Zhong <nju_y...@apache.org> Authored: Wed Aug 30 18:04:09 2017 +0800 Committer: Zhong <nju_y...@apache.org> Committed: Wed Aug 30 18:04:09 2017 +0800 ---------------------------------------------------------------------- .../kylin/cube/cuboid/TreeCuboidScheduler.java | 3 +- .../engine/mr/common/CuboidStatsReaderUtil.java | 10 +- .../kylin/rest/controller/CubeController.java | 94 +++++++++++++- .../kylin/rest/response/CuboidTreeResponse.java | 123 +++++++++++++++++++ .../apache/kylin/rest/service/CubeService.java | 60 +++++++++ 5 files changed, 279 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0b12df77/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java index 0cf770d..8252e6c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java @@ -19,7 +19,6 @@ package org.apache.kylin.cube.cuboid; import java.io.PrintWriter; -import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -243,7 +242,7 @@ public class TreeCuboidScheduler extends CuboidScheduler { @JsonIgnore int level; @JsonProperty("children") - List<TreeNode> children = new ArrayList<>(); + List<TreeNode> children = Lists.newArrayList(); public long getCuboidId() { return cuboidId; http://git-wip-us.apache.org/repos/asf/kylin/blob/0b12df77/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java index bfb37ce..604f600 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java @@ -38,9 +38,13 @@ public class CuboidStatsReaderUtil { private static final Logger logger = LoggerFactory.getLogger(CuboidStatsReaderUtil.class); - public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance) - throws IOException { - Map<Long, Long> statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst(); + public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance) { + Map<Long, Long> statisticsMerged = null; + try { + statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst(); + } catch (IOException e) { + logger.warn("Fail to read statistics for cube " + cubeInstance.getName() + " due to " + e); + } return statisticsMerged.isEmpty() ? null : statisticsMerged; } http://git-wip-us.apache.org/repos/asf/kylin/blob/0b12df77/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index c415212..3118c59 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -38,11 +38,14 @@ import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.cube.cuboid.CuboidScheduler; +import org.apache.kylin.cube.cuboid.TreeCuboidScheduler; import org.apache.kylin.cube.model.CubeBuildTypeEnum; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.RowKeyColDesc; import org.apache.kylin.dimension.DimensionEncodingFactory; import org.apache.kylin.engine.EngineFactory; +import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil; import org.apache.kylin.job.JobInstance; import org.apache.kylin.job.JoinedFlatTable; import org.apache.kylin.job.exception.JobException; @@ -61,6 +64,7 @@ import org.apache.kylin.rest.request.JobBuildRequest; import org.apache.kylin.rest.request.JobBuildRequest2; import org.apache.kylin.rest.request.JobOptimizeRequest; import org.apache.kylin.rest.request.SQLRequest; +import org.apache.kylin.rest.response.CuboidTreeResponse; import org.apache.kylin.rest.response.GeneralResponse; import org.apache.kylin.rest.response.HBaseResponse; import org.apache.kylin.rest.service.CubeService; @@ -763,20 +767,81 @@ public class CubeController extends BasicController { } } + @RequestMapping(value = "/{cubeName}/cuboids/current", method = RequestMethod.GET) + @ResponseBody + public CuboidTreeResponse getCurrentCuboids(@PathVariable String cubeName) { + CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); + if (cube == null) { + logger.error("Get cube: [" + cubeName + "] failed when get current cuboids"); + throw new BadRequestException("Get cube: [" + cubeName + "] failed when get current cuboids"); + } + // The cuboid tree displayed should be consistent with the current one + CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + Map<Long, Long> cuboidStatsMap = cube.getCuboids(); + if (cuboidStatsMap == null) { + cuboidStatsMap = CuboidStatsReaderUtil.readCuboidStatsFromCube(cuboidScheduler.getAllCuboidIds(), cube); + } + + Map<Long, Long> hitFrequencyMap = null; + Map<Long, Long> queryMatchMap = null; + try { + hitFrequencyMap = getTargetCuboidHitFrequency(cubeName); + queryMatchMap = getCuboidQueryMatchCount(cubeName); + } catch (Exception e) { + logger.warn("Fail to query on system cube due to " + e); + } + + Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds(); + return cubeService.getCuboidTreeResponse(cuboidScheduler, cuboidStatsMap, hitFrequencyMap, queryMatchMap, + currentCuboidSet); + } + + @RequestMapping(value = "/{cubeName}/cuboids/recommend", method = RequestMethod.GET) + @ResponseBody + public CuboidTreeResponse getRecommendCuboids(@PathVariable String cubeName) throws IOException { + CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); + if (cube == null) { + logger.error("Get cube: [" + cubeName + "] failed when get recommend cuboids"); + throw new BadRequestException("Get cube: [" + cubeName + "] failed when get recommend cuboids"); + } + Map<Long, Long> recommendCuboidStatsMap = getRecommendCuboidList(cube); + if (recommendCuboidStatsMap == null || recommendCuboidStatsMap.isEmpty()) { + return new CuboidTreeResponse(); + } + CuboidScheduler cuboidScheduler = new TreeCuboidScheduler(cube.getDescriptor(), + Lists.newArrayList(recommendCuboidStatsMap.keySet()), + new TreeCuboidScheduler.CuboidCostComparator(recommendCuboidStatsMap)); + + // Get cuboid target info for displaying heat map of cuboid hit + Map<Long, Long> displayHitFrequencyMap = getTargetCuboidHitFrequency(cubeName); + // Get exactly matched cuboid query count + Map<Long, Long> queryMatchMap = getCuboidQueryMatchCount(cubeName); + + Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds(); + return cubeService.getCuboidTreeResponse(cuboidScheduler, recommendCuboidStatsMap, displayHitFrequencyMap, + queryMatchMap, currentCuboidSet); + } + private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws IOException { // Get cuboid source info - Map<Long, Long> optimizeHitFrequencyMap = getCuboidHitFrequency(cube.getName(), true); + Map<Long, Long> optimizeHitFrequencyMap = getSourceCuboidHitFrequency(cube.getName()); Map<Long, Map<Long, Long>> rollingUpCountSourceMap = getCuboidRollingUpCount(cube.getName()); return cubeService.getRecommendCuboidStatistics(cube, optimizeHitFrequencyMap, rollingUpCountSourceMap); } - private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) { + private Map<Long, Long> getSourceCuboidHitFrequency(String cubeName) { + return getCuboidHitFrequency(cubeName, true); + } + + private Map<Long, Long> getTargetCuboidHitFrequency(String cubeName) { + return getCuboidHitFrequency(cubeName, false); + } + + private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean ifSource) { SQLRequest sqlRequest = new SQLRequest(); sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT); - String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString(); - if (!isCuboidSource) { - cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString(); - } + String cuboidColumn = ifSource ? QueryCubePropertyEnum.CUBOID_SOURCE.toString() + : QueryCubePropertyEnum.CUBOID_TARGET.toString(); String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(); String table = cubeService.getMetricsManager() .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube()); @@ -806,6 +871,23 @@ public class CubeController extends BasicController { return cubeService.formatRollingUpCount(orgRollingUpCount); } + private Map<Long, Long> getCuboidQueryMatchCount(String cubeName) { + SQLRequest sqlRequest = new SQLRequest(); + sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT); + String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString(); + String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(); + String table = cubeService.getMetricsManager() + .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube()); + String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ") " // + + "from " + table // + + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' and " + + QueryCubePropertyEnum.IF_MATCH.toString() + " = true " // + + "group by " + cuboidSource; + sqlRequest.setSql(sql); + List<List<String>> orgMatchHitFrequency = queryService.queryWithoutSecure(sqlRequest).getResults(); + return cubeService.formatQueryCount(orgMatchHitFrequency); + } + /** * Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming * source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point. http://git-wip-us.apache.org/repos/asf/kylin/blob/0b12df77/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java new file mode 100644 index 0000000..b416084 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.response; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; + +public class CuboidTreeResponse implements Serializable { + + private static final long serialVersionUID = 2835980715891990832L; + + private NodeInfo root; + + public NodeInfo getRoot() { + return root; + } + + public void setRoot(NodeInfo root) { + this.root = root; + } + + public static class NodeInfo { + @JsonProperty("cuboid_id") + private Long id; + @JsonProperty("name") + private String name; + @JsonProperty("query_count") + private Long queryCount; + @JsonProperty("query_rate") + private Float queryRate; + @JsonProperty("exactly_match_count") + private Long exactlyMatchCount; + @JsonProperty("row_count") + private Long rowCount; + @JsonProperty("existed") + private Boolean existed; + @JsonProperty("children") + List<NodeInfo> children = Lists.newArrayList(); + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Long getQueryCount() { + return queryCount; + } + + public void setQueryCount(Long queryCount) { + this.queryCount = queryCount; + } + + public Float getQueryRate() { + return queryRate; + } + + public void setQueryRate(Float queryRate) { + this.queryRate = queryRate; + } + + public Long getExactlyMatchCount() { + return exactlyMatchCount; + } + + public void setExactlyMatchCount(Long exactlyMatchCount) { + this.exactlyMatchCount = exactlyMatchCount; + } + + public Long getRowCount() { + return rowCount; + } + + public void setRowCount(Long rowCount) { + this.rowCount = rowCount; + } + + public Boolean getExisted() { + return existed; + } + + public void setExisted(Boolean existed) { + this.existed = existed; + } + + public void addChild(NodeInfo child) { + this.children.add(child); + } + + public List<NodeInfo> getChildren() { + return children; + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/0b12df77/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index fdf7bec..7268209 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -25,6 +25,7 @@ import java.util.Date; import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.WeakHashMap; import org.apache.commons.lang.StringUtils; @@ -35,7 +36,9 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.CubeUpdate; +import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.cuboid.CuboidCLI; +import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.engine.EngineFactory; import org.apache.kylin.engine.mr.CubingJob; @@ -57,6 +60,8 @@ import org.apache.kylin.rest.exception.ForbiddenException; import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.request.MetricsRequest; +import org.apache.kylin.rest.response.CuboidTreeResponse; +import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo; import org.apache.kylin.rest.response.HBaseResponse; import org.apache.kylin.rest.response.MetricsResponse; import org.apache.kylin.rest.security.AclPermission; @@ -768,6 +773,61 @@ public class CubeService extends BasicService { return formattedRollingUpCount; } + public CuboidTreeResponse getCuboidTreeResponse(CuboidScheduler cuboidScheduler, Map<Long, Long> rowCountMap, + Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap, Set<Long> currentCuboidSet) { + long baseCuboidId = cuboidScheduler.getBaseCuboidId(); + int dimensionCount = Long.bitCount(baseCuboidId); + + // get cube query count total + long cubeQueryCount = 0L; + if (hitFrequencyMap != null) { + for (long queryCount : hitFrequencyMap.values()) { + cubeQueryCount += queryCount; + } + } + + NodeInfo root = generateNodeInfo(baseCuboidId, dimensionCount, cubeQueryCount, rowCountMap, hitFrequencyMap, + queryMatchMap, currentCuboidSet); + + List<NodeInfo> nodeQueue = Lists.newLinkedList(); + nodeQueue.add(root); + while (!nodeQueue.isEmpty()) { + NodeInfo parentNode = nodeQueue.remove(0); + for (long childId : cuboidScheduler.getSpanningCuboid(parentNode.getId())) { + NodeInfo childNode = generateNodeInfo(childId, dimensionCount, cubeQueryCount, rowCountMap, + hitFrequencyMap, queryMatchMap, currentCuboidSet); + parentNode.addChild(childNode); + nodeQueue.add(childNode); + } + } + + CuboidTreeResponse result = new CuboidTreeResponse(); + result.setRoot(root); + return result; + } + + private NodeInfo generateNodeInfo(long cuboidId, int dimensionCount, long cubeQueryCount, + Map<Long, Long> rowCountMap, Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap, + Set<Long> currentCuboidSet) { + Long queryCount = hitFrequencyMap == null || hitFrequencyMap.get(cuboidId) == null ? 0L + : hitFrequencyMap.get(cuboidId); + float queryRate = cubeQueryCount <= 0 ? 0 : queryCount.floatValue() / cubeQueryCount; + long queryExactlyMatchCount = queryMatchMap == null || queryMatchMap.get(cuboidId) == null ? 0L + : queryMatchMap.get(cuboidId); + boolean ifExist = currentCuboidSet.contains(cuboidId); + long rowCount = rowCountMap == null ? 0L : rowCountMap.get(cuboidId); + + NodeInfo node = new NodeInfo(); + node.setId(cuboidId); + node.setName(Cuboid.getDisplayName(cuboidId, dimensionCount)); + node.setQueryCount(queryCount); + node.setQueryRate(queryRate); + node.setExactlyMatchCount(queryExactlyMatchCount); + node.setExisted(ifExist); + node.setRowCount(rowCount); + return node; + } + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')") public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
