KYLIN-2632 Refactor REST API and service
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2471d5e5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2471d5e5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2471d5e5 Branch: refs/heads/master Commit: 2471d5e56219d7c51342953ae37eb77370ae1e58 Parents: 0a0edfe Author: Luwei-Chen <chenlu...@apache.org> Authored: Tue May 23 18:17:29 2017 +0800 Committer: liyang-gmt8 <liy...@apache.org> Committed: Tue May 23 18:21:27 2017 +0800 ---------------------------------------------------------------------- .../rest/controller2/CubeControllerV2.java | 74 +-------- .../rest/controller2/DiagnosisControllerV2.java | 21 ++- .../rest/controller2/ModelControllerV2.java | 63 +++++--- .../rest/controller2/ProjectControllerV2.java | 109 +------------ .../rest/response/CubeInstanceResponse.java | 23 ++- .../rest/response/DataModelDescResponse.java | 23 ++- .../apache/kylin/rest/service/CacheService.java | 14 +- .../kylin/rest/service/CubeServiceV2.java | 159 ++++++------------ .../kylin/rest/service/ModelServiceV2.java | 161 +++++++------------ .../kylin/rest/service/ProjectServiceV2.java | 91 +++++++++++ 10 files changed, 319 insertions(+), 419 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java index 05d99f5..a9e00ba 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java @@ -21,6 +21,7 @@ package org.apache.kylin.rest.controller2; import java.io.IOException; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,6 +51,7 @@ import org.apache.kylin.rest.request.CubeRequest; import org.apache.kylin.rest.request.JobBuildRequest; import org.apache.kylin.rest.request.JobBuildRequest2; import org.apache.kylin.rest.response.CubeInstanceResponse; +import org.apache.kylin.rest.response.CubeInstanceResponse.CubeComparator; import org.apache.kylin.rest.response.EnvelopeResponse; import org.apache.kylin.rest.response.GeneralResponse; import org.apache.kylin.rest.response.HBaseResponse; @@ -74,7 +76,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.google.common.collect.Lists; @@ -149,6 +150,8 @@ public class CubeControllerV2 extends BasicController { cubeInstanceResponses.add(cubeInstanceResponse); } + CubeComparator cubeComparator = new CubeComparator(); + Collections.sort(cubeInstanceResponses, cubeComparator); data.put("cubes", cubeInstanceResponses); data.put("size", cubes.size()); @@ -468,58 +471,6 @@ public class CubeControllerV2 extends BasicController { } /** - * update CubDesc - * - * @return Table metadata array - * @throws JsonProcessingException - * @throws IOException - */ - - @RequestMapping(value = "", method = { RequestMethod.PUT }, produces = { "application/vnd.apache.kylin-v2+json" }) - @ResponseBody - public EnvelopeResponse updateCubeDescV2(@RequestHeader("Accept-Language") String lang, @RequestBody CubeRequest cubeRequest) throws IOException { - MsgPicker.setMsg(lang); - - CubeDesc desc = deserializeCubeDescV2(cubeRequest); - cubeServiceV2.validateCubeDesc(desc, false); - - boolean createNew = cubeServiceV2.unifyCubeDesc(desc, false); - - String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject(); - - desc = cubeServiceV2.updateCubeToResourceStore(desc, projectName, createNew, false); - - String descData = JsonUtil.writeValueAsIndentString(desc); - GeneralResponse data = new GeneralResponse(); - data.setProperty("uuid", desc.getUuid()); - data.setProperty("cubeDescData", descData); - - return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, data, ""); - } - - @RequestMapping(value = "/draft", method = { RequestMethod.PUT }, produces = { "application/vnd.apache.kylin-v2+json" }) - @ResponseBody - public EnvelopeResponse updateCubeDescDraftV2(@RequestHeader("Accept-Language") String lang, @RequestBody CubeRequest cubeRequest) throws IOException { - MsgPicker.setMsg(lang); - - CubeDesc desc = deserializeCubeDescV2(cubeRequest); - cubeServiceV2.validateCubeDesc(desc, true); - - boolean createNew = cubeServiceV2.unifyCubeDesc(desc, true); - - String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject(); - - desc = cubeServiceV2.updateCubeToResourceStore(desc, projectName, createNew, true); - - String descData = JsonUtil.writeValueAsIndentString(desc); - GeneralResponse data = new GeneralResponse(); - data.setProperty("uuid", desc.getUuid()); - data.setProperty("cubeDescData", descData); - - return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, data, ""); - } - - /** * get Hbase Info * * @return true @@ -704,23 +655,6 @@ public class CubeControllerV2 extends BasicController { return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, cubeServiceV2.checkNameAvailability(cubeName), ""); } - private CubeDesc deserializeCubeDescV2(CubeRequest cubeRequest) throws IOException { - Message msg = MsgPicker.getMsg(); - - CubeDesc desc = null; - try { - logger.debug("Saving cube " + cubeRequest.getCubeDescData()); - desc = JsonUtil.readValue(cubeRequest.getCubeDescData(), CubeDesc.class); - } catch (JsonParseException e) { - logger.error("The cube definition is not valid.", e); - throw new BadRequestException(msg.getINVALID_CUBE_DEFINITION()); - } catch (JsonMappingException e) { - logger.error("The cube definition is not valid.", e); - throw new BadRequestException(msg.getINVALID_CUBE_DEFINITION()); - } - return desc; - } - private AggregationGroup deserializeAggregationGroupV2(String aggregationGroupStr) throws IOException { AggregationGroup aggreationGroup = null; try { http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/controller2/DiagnosisControllerV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/DiagnosisControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/DiagnosisControllerV2.java index 290e130..c778026 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller2/DiagnosisControllerV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/DiagnosisControllerV2.java @@ -28,11 +28,13 @@ import javax.servlet.http.HttpServletResponse; import org.apache.kylin.metadata.badquery.BadQueryEntry; import org.apache.kylin.metadata.badquery.BadQueryHistory; +import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.rest.controller.BasicController; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.response.EnvelopeResponse; import org.apache.kylin.rest.response.ResponseCode; import org.apache.kylin.rest.service.DiagnosisService; +import org.apache.kylin.rest.service.ProjectServiceV2; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -57,19 +59,30 @@ public class DiagnosisControllerV2 extends BasicController { @Qualifier("diagnosisService") private DiagnosisService dgService; + @Autowired + @Qualifier("projectServiceV2") + private ProjectServiceV2 projectServiceV2; + /** * Get bad query history */ - @RequestMapping(value = "/{project}/sql", method = { RequestMethod.GET }, produces = { "application/vnd.apache.kylin-v2+json" }) + @RequestMapping(value = "/sql", method = { RequestMethod.GET }, produces = { "application/vnd.apache.kylin-v2+json" }) @ResponseBody - public EnvelopeResponse getBadQuerySqlV2(@RequestHeader("Accept-Language") String lang, @PathVariable String project, @RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset, @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize) throws IOException { + public EnvelopeResponse getBadQuerySqlV2(@RequestHeader("Accept-Language") String lang, @RequestParam(value = "project", required = false) String project, @RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset, @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize) throws IOException { MsgPicker.setMsg(lang); HashMap<String, Object> data = new HashMap<String, Object>(); List<BadQueryEntry> badEntry = Lists.newArrayList(); - BadQueryHistory badQueryHistory = dgService.getProjectBadQueryHistory(project); - badEntry.addAll(badQueryHistory.getEntries()); + if (project != null) { + BadQueryHistory badQueryHistory = dgService.getProjectBadQueryHistory(project); + badEntry.addAll(badQueryHistory.getEntries()); + } else { + for (ProjectInstance projectInstance : projectServiceV2.getReadableProjects()) { + BadQueryHistory badQueryHistory = dgService.getProjectBadQueryHistory(projectInstance.getName()); + badEntry.addAll(badQueryHistory.getEntries()); + } + } int offset = pageOffset * pageSize; int limit = pageSize; http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java index 7dc2be7..db70eca 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java @@ -18,11 +18,18 @@ package org.apache.kylin.rest.controller2; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.ResourceStore.Checkpoint; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.metadata.MetadataManager; @@ -35,9 +42,11 @@ import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.request.ModelRequest; import org.apache.kylin.rest.response.DataModelDescResponse; +import org.apache.kylin.rest.response.DataModelDescResponse.ModelComparator; import org.apache.kylin.rest.response.EnvelopeResponse; import org.apache.kylin.rest.response.GeneralResponse; import org.apache.kylin.rest.response.ResponseCode; +import org.apache.kylin.rest.service.CacheService; import org.apache.kylin.rest.service.ModelServiceV2; import org.apache.kylin.rest.service.ProjectServiceV2; import org.slf4j.Logger; @@ -53,12 +62,9 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.google.common.collect.Sets; /** * ModelController is defined as Restful API entrance for UI. @@ -80,6 +86,10 @@ public class ModelControllerV2 extends BasicController { @Qualifier("projectServiceV2") private ProjectServiceV2 projectServiceV2; + @Autowired + @Qualifier("cacheService") + private CacheService cacheService; + @RequestMapping(value = "", method = { RequestMethod.GET }, produces = { "application/vnd.apache.kylin-v2+json" }) @ResponseBody public EnvelopeResponse getModelsPaging(@RequestHeader("Accept-Language") String lang, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset, @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize) throws IOException { @@ -111,6 +121,8 @@ public class ModelControllerV2 extends BasicController { dataModelDescResponses.add(dataModelDescResponse); } + ModelComparator modelComparator = new ModelComparator(); + Collections.sort(dataModelDescResponses, modelComparator); data.put("models", dataModelDescResponses); data.put("size", models.size()); @@ -121,17 +133,24 @@ public class ModelControllerV2 extends BasicController { @ResponseBody public EnvelopeResponse updateModelDescV2(@RequestHeader("Accept-Language") String lang, @RequestBody ModelRequest modelRequest) throws IOException { MsgPicker.setMsg(lang); - Message msg = MsgPicker.getMsg(); - //Update Model DataModelDesc modelDesc = deserializeDataModelDescV2(modelRequest); modelServiceV2.validateModelDesc(modelDesc); - boolean createNew = modelServiceV2.unifyModelDesc(modelDesc, false); - String projectName = (null == modelRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : modelRequest.getProject(); - modelDesc = modelServiceV2.updateModelToResourceStore(modelDesc, projectName, createNew, false); + ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); + Checkpoint cp = store.checkpoint(); + try { + boolean createNew = modelServiceV2.unifyModelDesc(modelDesc, false); + modelDesc = modelServiceV2.updateModelToResourceStore(modelDesc, projectName, createNew, false); + } catch (Exception ex) { + cp.rollback(); + cacheService.wipeAllCache(); + throw ex; + } finally { + cp.close(); + } String descData = JsonUtil.writeValueAsIndentString(modelDesc); GeneralResponse data = new GeneralResponse(); @@ -145,16 +164,24 @@ public class ModelControllerV2 extends BasicController { @ResponseBody public EnvelopeResponse updateModelDescDraftV2(@RequestHeader("Accept-Language") String lang, @RequestBody ModelRequest modelRequest) throws IOException { MsgPicker.setMsg(lang); - Message msg = MsgPicker.getMsg(); DataModelDesc modelDesc = deserializeDataModelDescV2(modelRequest); modelServiceV2.validateModelDesc(modelDesc); - boolean createNew = modelServiceV2.unifyModelDesc(modelDesc, true); - String projectName = (null == modelRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : modelRequest.getProject(); - modelDesc = modelServiceV2.updateModelToResourceStore(modelDesc, projectName, createNew, true); + ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); + Checkpoint cp = store.checkpoint(); + try { + boolean createNew = modelServiceV2.unifyModelDesc(modelDesc, true); + modelDesc = modelServiceV2.updateModelToResourceStore(modelDesc, projectName, createNew, true); + } catch (Exception ex) { + cp.rollback(); + cacheService.wipeAllCache(); + throw ex; + } finally { + cp.close(); + } String descData = JsonUtil.writeValueAsIndentString(modelDesc); GeneralResponse data = new GeneralResponse(); http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java index 3ab0d66..841d5a0 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java @@ -19,16 +19,12 @@ package org.apache.kylin.rest.controller2; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.persistence.AclEntity; import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.metadata.project.ProjectInstance; -import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.controller.BasicController; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.msg.Message; @@ -36,19 +32,11 @@ import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.request.ProjectRequest; import org.apache.kylin.rest.response.EnvelopeResponse; import org.apache.kylin.rest.response.ResponseCode; -import org.apache.kylin.rest.service.AccessService; -import org.apache.kylin.rest.service.CubeServiceV2; import org.apache.kylin.rest.service.ProjectServiceV2; -import org.apache.kylin.rest.util.AclUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.security.acls.domain.GrantedAuthoritySid; -import org.springframework.security.acls.domain.PrincipalSid; -import org.springframework.security.acls.model.AccessControlEntry; -import org.springframework.security.acls.model.Acl; -import org.springframework.security.core.userdetails.UserDetails; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -72,17 +60,6 @@ public class ProjectControllerV2 extends BasicController { @Qualifier("projectServiceV2") private ProjectServiceV2 projectServiceV2; - @Autowired - @Qualifier("accessService") - private AccessService accessService; - - @Autowired - private AclUtil aclUtil; - - @Autowired - @Qualifier("cubeMgmtServiceV2") - private CubeServiceV2 cubeServiceV2; - @RequestMapping(value = "", method = { RequestMethod.GET }, produces = { "application/vnd.apache.kylin-v2+json" }) @ResponseBody public EnvelopeResponse getProjectsV2(@RequestHeader("Accept-Language") String lang, @RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset, @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize) { @@ -101,93 +78,10 @@ public class ProjectControllerV2 extends BasicController { HashMap<String, Object> data = new HashMap<String, Object>(); - List<ProjectInstance> readableProjects = new ArrayList<ProjectInstance>(); + List<ProjectInstance> readableProjects = projectServiceV2.getReadableProjects(); int offset = pageOffset * pageSize; int limit = pageSize; - //list all projects first - List<ProjectInstance> projectInstances = projectServiceV2.getProjectManager().listAllProjects(); - - if (projectInstances.size() <= offset) { - offset = projectInstances.size(); - limit = 0; - } - - if ((projectInstances.size() - offset) < limit) { - limit = projectInstances.size() - offset; - } - - //get user infomation - UserDetails userDetails = aclUtil.getCurrentUser(); - String userName = userDetails.getUsername(); - - //check if ROLE_ADMIN return all,also get user role list - List<String> userAuthority = aclUtil.getAuthorityList(); - for (String auth : userAuthority) { - if (auth.equals(Constant.ROLE_ADMIN)) { - data.put("readableProjects", projectInstances.subList(offset, offset + limit)); - data.put("size", projectInstances.size()); - return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, data, ""); - } - } - - for (ProjectInstance projectInstance : projectInstances) { - if (projectInstance == null) { - continue; - } - - boolean hasProjectPermission = false; - AclEntity ae = accessService.getAclEntity("ProjectInstance", projectInstance.getId()); - Acl projectAcl = accessService.getAcl(ae); - //project no Acl info will be skipped - if (projectAcl != null) { - - //project owner has permission - if (((PrincipalSid) projectAcl.getOwner()).getPrincipal().equals(userName)) { - readableProjects.add(projectInstance); - continue; - } - - //check project permission and role - for (AccessControlEntry ace : projectAcl.getEntries()) { - if (ace.getSid() instanceof PrincipalSid && ((PrincipalSid) ace.getSid()).getPrincipal().equals(userName)) { - hasProjectPermission = true; - readableProjects.add(projectInstance); - break; - - } else if (ace.getSid() instanceof GrantedAuthoritySid) { - String projectAuthority = ((GrantedAuthoritySid) ace.getSid()).getGrantedAuthority(); - if (userAuthority.contains(projectAuthority)) { - hasProjectPermission = true; - readableProjects.add(projectInstance); - break; - } - - } - - } - } - - if (!hasProjectPermission) { - List<CubeInstance> cubeInstances = cubeServiceV2.listAllCubes(projectInstance.getName()); - - for (CubeInstance cubeInstance : cubeInstances) { - if (cubeInstance == null) { - continue; - } - - if (aclUtil.isHasCubePermission(cubeInstance)) { - hasProjectPermission = true; - break; - } - } - if (hasProjectPermission) { - readableProjects.add(projectInstance); - } - } - - } - if (readableProjects.size() <= offset) { offset = readableProjects.size(); limit = 0; @@ -260,7 +154,6 @@ public class ProjectControllerV2 extends BasicController { @ResponseBody public void deleteProjectV2(@RequestHeader("Accept-Language") String lang, @PathVariable String projectName) throws IOException { MsgPicker.setMsg(lang); - Message msg = MsgPicker.getMsg(); ProjectInstance project = projectServiceV2.getProjectManager().getProject(projectName); projectServiceV2.deleteProject(projectName, project); http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java index 66878a4..ab59165 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java +++ b/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java @@ -18,13 +18,16 @@ package org.apache.kylin.rest.response; -import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Comparator; + import org.apache.kylin.cube.CubeInstance; +import com.fasterxml.jackson.annotation.JsonProperty; + /** * Created by luwei on 17-4-17. */ -public class CubeInstanceResponse extends CubeInstance{ +public class CubeInstanceResponse extends CubeInstance { public void setProject(String project) { this.project = project; @@ -69,4 +72,20 @@ public class CubeInstanceResponse extends CubeInstance{ setSegments(cubeInstance.getSegments()); setCreateTimeUTC(cubeInstance.getCreateTimeUTC()); } + + public static class CubeComparator implements Comparator<CubeInstanceResponse> { + @Override + public int compare(CubeInstanceResponse o1, CubeInstanceResponse o2) { + String name1 = o1.getName(), name2 = o2.getName(); + if (name1.endsWith("_draft")) { + name1 = name1.substring(0, name1.lastIndexOf("_draft")); + } + if (name2.endsWith("_draft")) { + name2 = name2.substring(0, name2.lastIndexOf("_draft")); + } + if (name1.equals(name2)) + return o1.getName().compareTo(o2.getName()); + return name1.compareTo(name2); + } + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java index 9fd0076..4f349a9 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java +++ b/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java @@ -18,11 +18,14 @@ package org.apache.kylin.rest.response; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Comparator; + import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.PartitionDesc; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; + /** * Created by luwei on 17-4-19. */ @@ -59,4 +62,20 @@ public class DataModelDescResponse extends DataModelDesc { setCapacity(dataModelDesc.getCapacity()); setComputedColumnDescs(dataModelDesc.getComputedColumnDescs()); } + + public static class ModelComparator implements Comparator<DataModelDescResponse> { + @Override + public int compare(DataModelDescResponse o1, DataModelDescResponse o2) { + String name1 = o1.getName(), name2 = o2.getName(); + if (name1.endsWith("_draft")) { + name1 = name1.substring(0, name1.lastIndexOf("_draft")); + } + if (name2.endsWith("_draft")) { + name2 = name2.substring(0, name2.lastIndexOf("_draft")); + } + if (name1.equals(name2)) + return o1.getName().compareTo(o2.getName()); + return name1.compareTo(name2); + } + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index 6f5e9f8..d26a224 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -18,7 +18,10 @@ package org.apache.kylin.rest.service; -import net.sf.ehcache.CacheManager; +import java.io.IOException; + +import javax.sql.DataSource; + import org.apache.commons.lang3.StringUtils; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; @@ -30,8 +33,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; -import javax.sql.DataSource; -import java.io.IOException; +import net.sf.ehcache.CacheManager; /** */ @@ -39,7 +41,7 @@ import java.io.IOException; public class CacheService extends BasicService { private static final Logger logger = LoggerFactory.getLogger(CacheService.class); - + private static QueryDataSource queryDataSource = new QueryDataSource(); @Autowired @@ -91,6 +93,10 @@ public class CacheService extends BasicService { this.cubeService = cubeService; } + public void wipeAllCache() { + annouceWipeCache("all", "update", "all"); + } + public void annouceWipeCache(String entity, String event, String cacheKey) { Broadcaster broadcaster = Broadcaster.getInstance(getConfig()); broadcaster.queue(entity, event, cacheKey); http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java index 2fae789..7103d24 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java @@ -78,7 +78,6 @@ public class CubeServiceV2 extends CubeService { @Qualifier("modelMgmtServiceV2") private ModelServiceV2 modelServiceV2; - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException { Message msg = MsgPicker.getMsg(); @@ -345,118 +344,75 @@ public class CubeServiceV2 extends CubeService { } public boolean unifyCubeDesc(CubeDesc desc, boolean isDraft) throws IOException { - Message msg = MsgPicker.getMsg(); - boolean createNew = false; - String cubeName = desc.getName(); - String originName = null; // for draft rename check - if (desc.getUuid() != null) { - originName = getNameByUuid(desc.getUuid()); + String name = desc.getName(); + if (isDraft) { + name += "_draft"; + desc.setName(name); + desc.setStatus(STATUS_DRAFT); + } else { + desc.setStatus(null); } - if (!isDraft) { // save as official cube - if (desc.getStatus() != null && desc.getStatus().equals(STATUS_DRAFT)) { // from draft - if (originName == null) { - throw new BadRequestException(msg.getORIGIN_CUBE_NOT_FOUND()); - } - originName = originName.substring(0, originName.lastIndexOf("_draft")); - if (!originName.equals(cubeName)) { // if rename draft - CubeDesc parentDesc = getCubeDescManager().getCubeDesc(originName); - if (parentDesc == null) { // only allow rename when official cube has not been saved - createNew = true; - deleteCubeByUuid(desc.getUuid()); - desc.setStatus(null); - desc.setLastModified(0); - desc.setUuid(UUID.randomUUID().toString()); - } else { - throw new BadRequestException(msg.getCUBE_RENAME()); - } - } else { // without rename draft - desc.setStatus(null); - CubeDesc parentDesc = getCubeDescManager().getCubeDesc(cubeName); - if (parentDesc == null) { // official cube doesn't exist, create new one - createNew = true; - desc.setLastModified(0); - desc.setUuid(UUID.randomUUID().toString()); - } else { // update existing - desc.setLastModified(parentDesc.getLastModified()); - desc.setUuid(parentDesc.getUuid()); - } - } - } else { // from official - if (originName == null) { // official cube doesn't exist, create new one - createNew = true; - desc.setLastModified(0); - desc.setUuid(UUID.randomUUID().toString()); - } else { - if (!originName.equals(cubeName)) { // do not allow official cube rename - throw new BadRequestException(msg.getCUBE_RENAME()); - } - } - } - } else { // save as draft model - if (desc.getStatus() == null) { // from official - cubeName += "_draft"; - desc.setName(cubeName); - desc.setStatus(STATUS_DRAFT); - CubeDesc draftDesc = getCubeDescManager().getCubeDesc(cubeName); - if (draftDesc == null) { - createNew = true; - desc.setLastModified(0); - desc.setUuid(UUID.randomUUID().toString()); - } else if (draftDesc.getStatus() != null && draftDesc.getStatus().equals(STATUS_DRAFT)) { // update existing - desc.setLastModified(draftDesc.getLastModified()); - desc.setUuid(draftDesc.getUuid()); - } else { // already exist an official draft with name ends with '_draft' - throw new BadRequestException(String.format(msg.getNON_DRAFT_CUBE_ALREADY_EXIST(), cubeName)); - } - } else if (desc.getStatus().equals(STATUS_DRAFT)) { // from draft - if (originName == null) { - throw new BadRequestException(msg.getORIGIN_CUBE_NOT_FOUND()); - } - originName = originName.substring(0, originName.lastIndexOf("_draft")); - if (!originName.equals(cubeName)) { // if rename draft - CubeDesc parentDesc = getCubeDescManager().getCubeDesc(originName); - if (parentDesc == null) { // only allow rename when official cube has not been saved - createNew = true; - deleteCubeByUuid(desc.getUuid()); - cubeName += "_draft"; - desc.setName(cubeName); - desc.setStatus(STATUS_DRAFT); - desc.setLastModified(0); - desc.setUuid(UUID.randomUUID().toString()); - } else { - throw new BadRequestException(msg.getMODEL_RENAME()); - } - } else { // without rename draft - cubeName += "_draft"; - desc.setName(cubeName); - } - } + if (desc.getUuid() == null) { + desc.setLastModified(0); + desc.setUuid(UUID.randomUUID().toString()); + return true; + } + + CubeDesc youngerSelf = killSameUuid(desc.getUuid(), name, isDraft); + if (youngerSelf != null) { + desc.setLastModified(youngerSelf.getLastModified()); + } else { + createNew = true; + desc.setLastModified(0); } + return createNew; } - public String getNameByUuid(String uuid) { + public CubeDesc killSameUuid(String uuid, String name, boolean isDraft) throws IOException { + Message msg = MsgPicker.getMsg(); + + CubeDesc youngerSelf = null, official = null; + boolean rename = false; List<CubeInstance> cubes = getCubeManager().listAllCubes(); for (CubeInstance cube : cubes) { - if (cube.getDescriptor().getUuid().equals(uuid)) { - return cube.getName(); + CubeDesc cubeDesc = cube.getDescriptor(); + if (cubeDesc.getUuid().equals(uuid)) { + boolean toDrop = true; + boolean sameStatus = sameStatus(cubeDesc.getStatus(), isDraft); + if (sameStatus && !cubeDesc.getName().equals(name)) { + rename = true; + } + if (sameStatus && cubeDesc.getName().equals(name)) { + youngerSelf = cubeDesc; + toDrop = false; + } + if (cubeDesc.getStatus() == null) { + official = cubeDesc; + toDrop = false; + } + if (toDrop) { + deleteCube(cube); + } } } - return null; + if (official != null && rename) { + throw new BadRequestException(msg.getCUBE_RENAME()); + } + return youngerSelf; } - public void deleteCubeByUuid(String uuid) throws IOException { - List<CubeInstance> cubes = getCubeManager().listAllCubes(); - for (CubeInstance cube : cubes) { - if (cube.getDescriptor().getUuid().equals(uuid)) { - deleteCube(cube); - } + public boolean sameStatus(String status, boolean isDraft) { + if (status == null || !status.equals(STATUS_DRAFT)) { + return !isDraft; + } else { + return isDraft; } } - public CubeDesc updateCubeToResourceStore(CubeDesc desc, String projectName, boolean createNew, boolean isDraft) throws IOException { + public CubeDesc updateCubeToResourceStore(CubeDesc desc, String projectName, boolean createNew) throws IOException { Message msg = MsgPicker.getMsg(); String cubeName = desc.getName(); @@ -483,15 +439,6 @@ public class CubeServiceV2 extends CubeService { throw new BadRequestException(String.format(msg.getBROKEN_CUBE_DESC(), cubeName)); } } - - if (!isDraft) { - String draftName = desc.getName() + "_draft"; - CubeInstance draftCube = getCubeManager().getCube(draftName); - if (null != draftCube && draftCube.getDescriptor().getStatus() != null && draftCube.getDescriptor().getStatus().equals(STATUS_DRAFT)) { - //drop draft Cube - deleteCube(draftCube); - } - } return desc; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java index 1628964..3e458ad 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java @@ -80,9 +80,11 @@ public class ModelServiceV2 extends ModelService { String owner = SecurityContextHolder.getContext().getAuthentication().getName(); createdDesc = getMetadataManager().createDataModelDesc(desc, projectName, owner); - accessService.init(createdDesc, AclPermission.ADMINISTRATION); - ProjectInstance project = getProjectManager().getProject(projectName); - accessService.inherit(createdDesc, project); + if (desc.getStatus() == null || !desc.getStatus().equals(STATUS_DRAFT)) { + accessService.init(createdDesc, AclPermission.ADMINISTRATION); + ProjectInstance project = getProjectManager().getProject(projectName); + accessService.inherit(createdDesc, project); + } return createdDesc; } @@ -208,115 +210,71 @@ public class ModelServiceV2 extends ModelService { } } - public boolean unifyModelDesc(DataModelDesc modelDesc, boolean isDraft) throws IOException { - Message msg = MsgPicker.getMsg(); - + public boolean unifyModelDesc(DataModelDesc desc, boolean isDraft) throws IOException { boolean createNew = false; - String modelName = modelDesc.getName(); - String originName = null; // for draft rename check - if (modelDesc.getUuid() != null) { - originName = getNameByUuid(modelDesc.getUuid()); + String name = desc.getName(); + if (isDraft) { + name += "_draft"; + desc.setName(name); + desc.setStatus(STATUS_DRAFT); + } else { + desc.setStatus(null); } - if (!isDraft) { // save as official model - if (modelDesc.getStatus() != null && modelDesc.getStatus().equals(STATUS_DRAFT)) { // from draft - if (originName == null) { - throw new BadRequestException(msg.getORIGIN_MODEL_NOT_FOUND()); - } - originName = originName.substring(0, originName.lastIndexOf("_draft")); - if (!originName.equals(modelName)) { // if rename draft - DataModelDesc parentDesc = getMetadataManager().getDataModelDesc(originName); - if (parentDesc == null) { // only allow rename when official model has not been saved - createNew = true; - dropModelByUuid(modelDesc.getUuid()); - modelDesc.setStatus(null); - modelDesc.setLastModified(0); - modelDesc.setUuid(UUID.randomUUID().toString()); - } else { - throw new BadRequestException(msg.getMODEL_RENAME()); - } - } else { // without rename draft - modelDesc.setStatus(null); - DataModelDesc parentDesc = getMetadataManager().getDataModelDesc(modelName); - if (parentDesc == null) { // official model doesn't exist, create new one - createNew = true; - modelDesc.setLastModified(0); - modelDesc.setUuid(UUID.randomUUID().toString()); - } else { // update existing - modelDesc.setLastModified(parentDesc.getLastModified()); - modelDesc.setUuid(parentDesc.getUuid()); - } - } - } else { // from official - if (originName == null) { // official model doesn't exist, create new one - createNew = true; - modelDesc.setLastModified(0); - modelDesc.setUuid(UUID.randomUUID().toString()); - } else { - if (!originName.equals(modelName)) { // do not allow official model rename - throw new BadRequestException(msg.getMODEL_RENAME()); - } - } - } - } else { // save as draft model - if (modelDesc.getStatus() == null) { // from official - modelName += "_draft"; - modelDesc.setName(modelName); - modelDesc.setStatus(STATUS_DRAFT); - DataModelDesc draftDesc = getMetadataManager().getDataModelDesc(modelName); - if (draftDesc == null) { // draft model doesn't exist, create new one - createNew = true; - modelDesc.setLastModified(0); - modelDesc.setUuid(UUID.randomUUID().toString()); - } else if (draftDesc.getStatus() != null && draftDesc.getStatus().equals(STATUS_DRAFT)) { // update existing - modelDesc.setLastModified(draftDesc.getLastModified()); - modelDesc.setUuid(draftDesc.getUuid()); - } else { // already exist an official draft with name ends with '_draft' - throw new BadRequestException(String.format(msg.getNON_DRAFT_MODEL_ALREADY_EXIST(), modelName)); - } - } else if (modelDesc.getStatus().equals(STATUS_DRAFT)) { // from draft - if (originName == null) { - throw new BadRequestException(msg.getORIGIN_MODEL_NOT_FOUND()); - } - originName = originName.substring(0, originName.lastIndexOf("_draft")); - if (!originName.equals(modelName)) { // if rename draft - DataModelDesc parentDesc = getMetadataManager().getDataModelDesc(originName); - if (parentDesc == null) { // only allow rename when official model has not been saved - createNew = true; - dropModelByUuid(modelDesc.getUuid()); - modelName += "_draft"; - modelDesc.setName(modelName); - modelDesc.setStatus(STATUS_DRAFT); - modelDesc.setLastModified(0); - modelDesc.setUuid(UUID.randomUUID().toString()); - } else { - throw new BadRequestException(msg.getMODEL_RENAME()); - } - } else { // without rename draft - modelName += "_draft"; - modelDesc.setName(modelName); - } - } + if (desc.getUuid() == null) { + desc.setLastModified(0); + desc.setUuid(UUID.randomUUID().toString()); + return true; + } + + DataModelDesc youngerSelf = killSameUuid(desc.getUuid(), name, isDraft); + if (youngerSelf != null) { + desc.setLastModified(youngerSelf.getLastModified()); + } else { + createNew = true; + desc.setLastModified(0); } + return createNew; } - public String getNameByUuid(String uuid) { + public DataModelDesc killSameUuid(String uuid, String name, boolean isDraft) throws IOException { + Message msg = MsgPicker.getMsg(); + + DataModelDesc youngerSelf = null, official = null; + boolean rename = false; List<DataModelDesc> models = getMetadataManager().getModels(); for (DataModelDesc model : models) { if (model.getUuid().equals(uuid)) { - return model.getName(); + boolean toDrop = true; + boolean sameStatus = sameStatus(model.getStatus(), isDraft); + if (sameStatus && !model.getName().equals(name)) { + rename = true; + } + if (sameStatus && model.getName().equals(name)) { + youngerSelf = model; + toDrop = false; + } + if (model.getStatus() == null) { + official = model; + toDrop = false; + } + if (toDrop) { + dropModel(model); + } } } - return null; + if (official != null && rename) { + throw new BadRequestException(msg.getMODEL_RENAME()); + } + return youngerSelf; } - public void dropModelByUuid(String uuid) throws IOException { - List<DataModelDesc> models = getMetadataManager().getModels(); - for (DataModelDesc model : models) { - if (model.getUuid().equals(uuid)) { - dropModel(model); - } + public boolean sameStatus(String status, boolean isDraft) { + if (status == null || !status.equals(STATUS_DRAFT)) { + return !isDraft; + } else { + return isDraft; } } @@ -339,13 +297,6 @@ public class ModelServiceV2 extends ModelService { throw new BadRequestException(String.format(msg.getBROKEN_MODEL_DESC(), modelDesc.getName())); } } - - if (!isDraft) { - DataModelDesc draftDesc = getMetadataManager().getDataModelDesc(modelDesc.getName() + "_draft"); - if (null != draftDesc && draftDesc.getStatus() != null && draftDesc.getStatus().equals(STATUS_DRAFT)) { - dropModel(draftDesc); - } - } return modelDesc; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java index fb1ac94..b05b18c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java @@ -19,21 +19,30 @@ package org.apache.kylin.rest.service; import java.io.IOException; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import org.apache.kylin.common.persistence.AclEntity; +import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.security.AclPermission; +import org.apache.kylin.rest.util.AclUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.acls.domain.GrantedAuthoritySid; +import org.springframework.security.acls.domain.PrincipalSid; +import org.springframework.security.acls.model.AccessControlEntry; +import org.springframework.security.acls.model.Acl; import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.userdetails.UserDetails; import org.springframework.stereotype.Component; /** @@ -48,6 +57,13 @@ public class ProjectServiceV2 extends ProjectService { @Qualifier("accessService") private AccessService accessService; + @Autowired + @Qualifier("cubeMgmtServiceV2") + private CubeServiceV2 cubeServiceV2; + + @Autowired + private AclUtil aclUtil; + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public ProjectInstance createProject(ProjectInstance newProject) throws IOException { Message msg = MsgPicker.getMsg(); @@ -85,5 +101,80 @@ public class ProjectServiceV2 extends ProjectService { return null; } + public List<ProjectInstance> getReadableProjects() { + List<ProjectInstance> readableProjects = new ArrayList<ProjectInstance>(); + + //list all projects first + List<ProjectInstance> projectInstances = getProjectManager().listAllProjects(); + + //get user infomation + UserDetails userDetails = aclUtil.getCurrentUser(); + String userName = userDetails.getUsername(); + //check if ROLE_ADMIN return all,also get user role list + List<String> userAuthority = aclUtil.getAuthorityList(); + for (String auth : userAuthority) { + if (auth.equals(Constant.ROLE_ADMIN)) { + return projectInstances; + } + } + + for (ProjectInstance projectInstance : projectInstances) { + if (projectInstance == null) { + continue; + } + + boolean hasProjectPermission = false; + AclEntity ae = accessService.getAclEntity("ProjectInstance", projectInstance.getId()); + Acl projectAcl = accessService.getAcl(ae); + //project no Acl info will be skipped + if (projectAcl != null) { + + //project owner has permission + if (((PrincipalSid) projectAcl.getOwner()).getPrincipal().equals(userName)) { + readableProjects.add(projectInstance); + continue; + } + + //check project permission and role + for (AccessControlEntry ace : projectAcl.getEntries()) { + if (ace.getSid() instanceof PrincipalSid && ((PrincipalSid) ace.getSid()).getPrincipal().equals(userName)) { + hasProjectPermission = true; + readableProjects.add(projectInstance); + break; + + } else if (ace.getSid() instanceof GrantedAuthoritySid) { + String projectAuthority = ((GrantedAuthoritySid) ace.getSid()).getGrantedAuthority(); + if (userAuthority.contains(projectAuthority)) { + hasProjectPermission = true; + readableProjects.add(projectInstance); + break; + } + + } + + } + } + + if (!hasProjectPermission) { + List<CubeInstance> cubeInstances = cubeServiceV2.listAllCubes(projectInstance.getName()); + + for (CubeInstance cubeInstance : cubeInstances) { + if (cubeInstance == null) { + continue; + } + + if (aclUtil.isHasCubePermission(cubeInstance)) { + hasProjectPermission = true; + break; + } + } + if (hasProjectPermission) { + readableProjects.add(projectInstance); + } + } + + } + return readableProjects; + } }
