put ext filter into project L2 cache
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0e662a35 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0e662a35 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0e662a35 Branch: refs/heads/1.5.x-HBase1.1.3 Commit: 0e662a350b88ad4b1194874af19f25590286a52a Parents: c86de1a Author: Hongbin Ma <mahong...@apache.org> Authored: Fri Apr 1 15:47:50 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Fri Apr 1 15:47:50 2016 +0800 ---------------------------------------------------------------------- .../metadata/filter/UDF/MassInTupleFilter.java | 1 + .../kylin/metadata/project/ProjectL2Cache.java | 37 +++++++++++++++----- .../kylin/metadata/project/ProjectManager.java | 9 +++++ .../controller/ExternalFilterController.java | 37 ++++++++++---------- .../kylin/rest/service/ExtFilterService.java | 21 ----------- 5 files changed, 58 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java index 0cb416c..5889c61 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java @@ -32,6 +32,7 @@ import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.function.Functions; import org.apache.kylin.metadata.model.ExternalFilterDesc; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java index e8004e1..3006773 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java @@ -25,6 +25,7 @@ import java.util.Set; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.ColumnDesc; +import org.apache.kylin.metadata.model.ExternalFilterDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TableDesc; @@ -59,6 +60,16 @@ class ProjectL2Cache { projectCaches.clear(); } + public ExternalFilterDesc getExternalFilterDesc(String project, String extFilterName) { + ProjectCache prjCache = getCache(project); + return prjCache.extFilters.get(extFilterName); + } + + public Map<String, ExternalFilterDesc> listExternalFilterDesc(String project) { + ProjectCache prjCache = getCache(project); + return Collections.unmodifiableMap(prjCache.extFilters); + } + public List<TableDesc> listDefinedTables(String project) { ProjectCache prjCache = getCache(project); List<TableDesc> result = Lists.newArrayListWithCapacity(prjCache.tables.size()); @@ -159,7 +170,7 @@ class ProjectL2Cache { private ProjectCache loadCache(String project) { logger.info("Loading L2 project cache for " + project); - ProjectCache result = new ProjectCache(project); + ProjectCache projectCache = new ProjectCache(project); ProjectInstance pi = mgr.getProject(project); @@ -171,30 +182,39 @@ class ProjectL2Cache { for (String tableName : pi.getTables()) { TableDesc tableDesc = metaMgr.getTableDesc(tableName); if (tableDesc != null) { - result.tables.put(tableDesc.getIdentity(), new TableCache(tableDesc)); + projectCache.tables.put(tableDesc.getIdentity(), new TableCache(tableDesc)); } else { logger.warn("Table '" + tableName + "' defined under project '" + project + "' is not found"); } } + for (String extFilterName : pi.getExtFilters()) { + ExternalFilterDesc filterDesc = metaMgr.getExtFilterDesc(extFilterName); + if (filterDesc != null) { + projectCache.extFilters.put(extFilterName, filterDesc); + } else { + logger.warn("External Filter '" + extFilterName + "' defined under project '" + project + "' is not found"); + } + } + RealizationRegistry registry = RealizationRegistry.getInstance(mgr.getConfig()); for (RealizationEntry entry : pi.getRealizationEntries()) { IRealization realization = registry.getRealization(entry.getType(), entry.getRealization()); if (realization != null) { - result.realizations.add(realization); + projectCache.realizations.add(realization); } else { logger.warn("Realization '" + entry + "' defined under project '" + project + "' is not found"); } } - for (IRealization realization : result.realizations) { - if (sanityCheck(result, realization)) { - mapTableToRealization(result, realization); - markExposedTablesAndColumns(result, realization); + for (IRealization realization : projectCache.realizations) { + if (sanityCheck(projectCache, realization)) { + mapTableToRealization(projectCache, realization); + markExposedTablesAndColumns(projectCache, realization); } } - return result; + return projectCache; } // check all columns reported by realization does exists @@ -257,6 +277,7 @@ class ProjectL2Cache { private Map<String, TableCache> tables = Maps.newHashMap(); private Set<TableDesc> exposedTables = Sets.newHashSet(); private Set<IRealization> realizations = Sets.newHashSet(); + private Map<String, ExternalFilterDesc> extFilters = Maps.newHashMap(); ProjectCache(String project) { this.project = project; http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index 65b3719..b42b7e2 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -21,6 +21,7 @@ package org.apache.kylin.metadata.project; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -352,6 +353,14 @@ public class ProjectManager { return projects; } + public ExternalFilterDesc getExternalFilterDesc(String project, String extFilter) { + return l2Cache.getExternalFilterDesc(project, extFilter); + } + + public Map<String, ExternalFilterDesc> listExternalFilterDescs(String project) { + return l2Cache.listExternalFilterDesc(project); + } + public List<TableDesc> listDefinedTables(String project) throws IOException { return l2Cache.listDefinedTables(norm(project)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java b/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java index b0b7569..5801eb5 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java @@ -18,6 +18,12 @@ package org.apache.kylin.rest.controller; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.metadata.model.ExternalFilterDesc; import org.apache.kylin.rest.request.ExternalFilterRequest; @@ -26,13 +32,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import com.google.common.collect.Lists; /** * @author jiazhong @@ -45,8 +52,6 @@ public class ExternalFilterController extends BasicController { @Autowired private ExtFilterService extFilterService; - - @RequestMapping(value = "/saveExtFilter", method = { RequestMethod.POST }) @ResponseBody public Map<String, String> saveExternalFilter(@RequestBody ExternalFilterRequest request) throws IOException { @@ -55,8 +60,8 @@ public class ExternalFilterController extends BasicController { ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), ExternalFilterDesc.class); desc.setUuid(UUID.randomUUID().toString()); extFilterService.saveExternalFilter(desc); - extFilterService.syncExtFilterToProject(new String[]{desc.getName()},filterProject); - result.put("success","true"); + extFilterService.syncExtFilterToProject(new String[] { desc.getName() }, filterProject); + result.put("success", "true"); return result; } @@ -66,7 +71,7 @@ public class ExternalFilterController extends BasicController { Map<String, String> result = new HashMap(); ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), ExternalFilterDesc.class); extFilterService.updateExternalFilter(desc); - result.put("success","true"); + result.put("success", "true"); return result; } @@ -74,13 +79,12 @@ public class ExternalFilterController extends BasicController { @ResponseBody public Map<String, String> unLoadHiveTables(@PathVariable String filter, @PathVariable String project) throws IOException { Map<String, String> result = new HashMap<String, String>(); - extFilterService.removeExtFilterFromProject(filter,project); + extFilterService.removeExtFilterFromProject(filter, project); extFilterService.removeExternalFilter(filter); result.put("success", "true"); return result; } - /** * Get available table list of the input database * @@ -90,12 +94,9 @@ public class ExternalFilterController extends BasicController { @RequestMapping(value = "", method = { RequestMethod.GET }) @ResponseBody public List<ExternalFilterDesc> getExternalFilters(@RequestParam(value = "project", required = true) String project) throws IOException { - List<ExternalFilterDesc> filterDescs; - filterDescs = extFilterService.listProjectFilters(project); + List<ExternalFilterDesc> filterDescs = Lists.newArrayList(); + filterDescs.addAll(extFilterService.getProjectManager().listExternalFilterDescs(project).values()); return filterDescs; } - - - } http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java b/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java index f41996d..6577a12 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java @@ -33,11 +33,6 @@ import org.springframework.stereotype.Component; import java.io.IOException; import java.util.List; -/** - * Stateless & lightweight service facade of cube management functions. - * - * @author jiazhong - */ @Component("extFilterService") public class ExtFilterService extends BasicService { private static final Logger logger = LoggerFactory.getLogger(ExtFilterService.class); @@ -47,22 +42,6 @@ public class ExtFilterService extends BasicService { private AccessService accessService; - - public List<ExternalFilterDesc> listProjectFilters(String project) throws IOException { - List<ExternalFilterDesc> filterDescs = Lists.newArrayList(); - if(project == null){ - return filterDescs; - } - ProjectInstance projectInstance = getProjectManager().getProject(project); - for(String filter:projectInstance.getExtFilters()){ - ExternalFilterDesc filterDesc = getMetadataManager().getExtFilterDesc(filter); - if(filterDesc!=null){ - filterDescs.add(filterDesc); - } - } - return filterDescs; - } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public void saveExternalFilter(ExternalFilterDesc desc) throws IOException { if(getMetadataManager().getExtFilterDesc(desc.getName()) != null){
