Repository: kylin Updated Branches: refs/heads/master 34cbd1813 -> a41a7e7d3
KYLIN-2999 One click migrate cube in web Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a41a7e7d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a41a7e7d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a41a7e7d Branch: refs/heads/master Commit: a41a7e7d3a97197a16360ab41e94d9ee5c77ebc6 Parents: 34cbd18 Author: kangkaisen <kangkai...@meituan.com> Authored: Tue Dec 5 20:37:08 2017 +0800 Committer: kangkaisen <kangkai...@meituan.com> Committed: Fri Jan 26 17:28:54 2018 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 25 +++++++++++++ .../kylin/rest/controller/CubeController.java | 7 ++++ .../apache/kylin/rest/service/CubeService.java | 39 ++++++++++++++++++++ webapp/app/js/controllers/cubes.js | 33 ++++++++++++++++- webapp/app/js/services/cubes.js | 3 +- webapp/app/js/services/kylinProperties.js | 14 ++++++- webapp/app/partials/cubes/cubes.html | 1 + 7 files changed, 117 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index cfe6fe2..4c8a3f8 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1572,4 +1572,29 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.metrics.subject-query-rpc", "METRICS_QUERY_RPC") + "_" + getKylinMetricsSubjectSuffix(); } + + // ============================================================================ + // tool + // ============================================================================ + public boolean isAllowAutoMigrateCube() { + return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.enabled", "false")); + } + + public boolean isAutoMigrateCubeCopyAcl() { + return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.copy-acl", "true")); + } + + public boolean isAutoMigrateCubePurge() { + return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.purge-src-cube", "true")); + } + + public String getAutoMigrateCubeSrcConfig() { + return getOptional("kylin.tool.auto-migrate-cube.src-config", ""); + } + + public String getAutoMigrateCubeDestConfig() { + return getOptional("kylin.tool.auto-migrate-cube.dest-config", ""); + } + + } http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index f6c2abf..89c11a8 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -1022,6 +1022,13 @@ public class CubeController extends BasicController { } } + @RequestMapping(value = "/{cube}/{project}/migrate", method = { RequestMethod.POST }) + @ResponseBody + public void migrateCube(@PathVariable String cube, @PathVariable String project) { + CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cube); + cubeService.migrateCube(cubeInstance, project); + } + public void setCubeService(CubeService cubeService) { this.cubeService = cubeService; } http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index a81d189..154354d 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -30,6 +30,7 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.RootPersistentEntity; +import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; @@ -41,6 +42,7 @@ import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.engine.EngineFactory; import org.apache.kylin.engine.mr.CubingJob; import org.apache.kylin.engine.mr.common.CuboidRecommenderUtil; +import org.apache.kylin.job.common.PatternedLogger; import org.apache.kylin.job.exception.JobException; import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableState; @@ -60,6 +62,7 @@ import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.exception.ForbiddenException; +import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.request.MetricsRequest; @@ -844,4 +847,40 @@ public class CubeService extends BasicService implements InitializingBean { Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException { return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap); } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") + public void migrateCube(CubeInstance cube, String projectName) { + KylinConfig config = cube.getConfig(); + try { + if (!config.isAllowAutoMigrateCube()) { + throw new InternalErrorException("This cube couldn't one click migrate cube, Please contact your ADMIN"); + } + + for (CubeSegment segment : cube.getSegments()) { + if (segment.getStatus() != SegmentStatusEnum.READY) { + throw new InternalErrorException("At least one segment is not in READY state. Please check whether there are Running or Error jobs."); + } + } + + String srcCfgUri = config.getAutoMigrateCubeSrcConfig(); + String dstCfgUri = config.getAutoMigrateCubeDestConfig(); + + String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true"); + String cmd = String.format(stringBuilder, KylinConfig.getKylinHome(), srcCfgUri, dstCfgUri, cube.getName(), + projectName, config.isAutoMigrateCubeCopyAcl(), config.isAutoMigrateCubePurge()); + + logger.info("cmd: " + cmd); + + CliCommandExecutor exec = new CliCommandExecutor(); + PatternedLogger patternedLogger = new PatternedLogger(logger); + + try { + exec.execute(cmd, patternedLogger); + } catch (IOException e) { + throw new InternalErrorException("Migrating cube fails, Please contact your ADMIN"); + } + } catch (Throwable e) { + throw new InternalErrorException(e.getMessage()); + } + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/webapp/app/js/controllers/cubes.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js index c123f1c..7094ff8 100644 --- a/webapp/app/js/controllers/cubes.js +++ b/webapp/app/js/controllers/cubes.js @@ -293,6 +293,35 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, }); }; + $scope.isAutoMigrateCubeEnabled = function(){ + return kylinConfig.isAutoMigrateCubeEnabled(); + }; + + $scope.migrateCube = function (cube) { + SweetAlert.swal({ + title: '', + text: "The cube will overwrite the same cube in prod env" + + "\nMigrating cube will elapse a couple of minutes." + + "\nPlease wait.", + type: '', + showCancelButton: true, + confirmButtonColor: '#DD6B55', + confirmButtonText: "Yes", + closeOnConfirm: true + }, function(isConfirm) { + if(isConfirm){ + loadingRequest.show(); + CubeService.autoMigrate({cubeId: cube.name, propName: $scope.projectModel.selectedProject}, {}, function (result) { + loadingRequest.hide(); + SweetAlert.swal('Success!', cube.name + ' migrate successfully!', 'success'); + },function(e){ + loadingRequest.hide(); + SweetAlert.swal('Migrate failed!', "Please contact your ADMIN.", 'error'); + }); + } + }); + }; + $scope.startJobSubmit = function (cube) { $scope.loadDetail(cube).then(function () { @@ -464,8 +493,6 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, }); }); } - - $scope.cubeEdit = function (cube) { $location.path("cubes/edit/" + cube.name); } @@ -494,6 +521,8 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, }); }) } + + }); http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/webapp/app/js/services/cubes.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js index 150d932..30c2a3f 100644 --- a/webapp/app/js/services/cubes.js +++ b/webapp/app/js/services/cubes.js @@ -77,6 +77,7 @@ KylinApp.factory('CubeService', ['$resource', function ($resource, config) { } } }, - optimize: {method: 'PUT', params: {action: 'optimize'}, isArray: false} + optimize: {method: 'PUT', params: {action: 'optimize'}, isArray: false}, + autoMigrate: {method: 'POST', params: {action: 'migrate'}, isArray: false} }); }]); http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/webapp/app/js/services/kylinProperties.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/services/kylinProperties.js b/webapp/app/js/services/kylinProperties.js index dc9b94d..ec23397 100644 --- a/webapp/app/js/services/kylinProperties.js +++ b/webapp/app/js/services/kylinProperties.js @@ -114,7 +114,7 @@ KylinApp.service('kylinConfig', function (AdminService, $log) { } return true; } - + this.isAdminExportAllowed = function(){ var status = this.getProperty("kylin.web.export-allow-admin").trim(); if(status!=='false'){ @@ -130,12 +130,13 @@ KylinApp.service('kylinConfig', function (AdminService, $log) { } return false; } - + this.getHiddenMeasures = function() { var hide_measures = this.getProperty("kylin.web.hide-measures").replace(/\s/g,"").toUpperCase(); return hide_measures.split(",") } + this.getQueryTimeout = function () { var queryTimeout = parseInt(this.getProperty("kylin.web.query-timeout")); if (isNaN(queryTimeout)) { @@ -147,5 +148,14 @@ KylinApp.service('kylinConfig', function (AdminService, $log) { this.isInitialized = function() { return angular.isString(_config); } + + this.isAutoMigrateCubeEnabled = function(){ + var status = this.getProperty("kylin.tool.auto-migrate-cube.enabled").trim(); + if(status && status =='true'){ + return true; + } + return false; + } + }); http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/webapp/app/partials/cubes/cubes.html ---------------------------------------------------------------------- diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html index 94dc3a6..fb98fe3 100644 --- a/webapp/app/partials/cubes/cubes.html +++ b/webapp/app/partials/cubes/cubes.html @@ -99,6 +99,7 @@ <li ng-if="cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"><a ng-click="enable(cube)">Enable</a></li> <li ng-if="cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"><a ng-click="purge(cube)">Purge</a></li> <li ng-if="cube.status!='DESCBROKEN' && (userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"><a ng-click="cloneCube(cube)">Clone</a></li> + <li ng-if="cube.status=='READY' && isAutoMigrateCubeEnabled() && (userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask)) "><a ng-click="migrateCube(cube)">Migrate</a></li> </ul> <ul ng-if="!(userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask))" class="dropdown-menu" role="menu">
