KYLIN-2187 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73ec4a4c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73ec4a4c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73ec4a4c Branch: refs/heads/master Commit: 73ec4a4c20f8d8cf372a15d2100c7bfd919b4ba5 Parents: 6b35e5e Author: Li Yang <[email protected]> Authored: Wed Nov 16 11:07:31 2016 +0800 Committer: Li Yang <[email protected]> Committed: Wed Nov 16 11:07:31 2016 +0800 ---------------------------------------------------------------------- .../kylin/metadata/model/TableExtDesc.java | 20 +----- .../apache/kylin/rest/service/BasicService.java | 70 -------------------- .../apache/kylin/rest/service/CubeService.java | 11 +-- .../apache/kylin/rest/service/JobService.java | 59 +++++++++++++++++ 4 files changed, 69 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec4a4c/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java index fc2edd4..0086917 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java @@ -26,8 +26,6 @@ import java.util.Map; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonBackReference; @@ -36,18 +34,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; @SuppressWarnings("serial") @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class TableExtDesc extends RootPersistentEntity implements ISourceAware { - - private static final Logger logger = LoggerFactory.getLogger(TableExtDesc.class); +public class TableExtDesc extends RootPersistentEntity { @JsonProperty("table_name") private String tableName; @JsonProperty("last_build_job_id") private String jodID; - @JsonProperty("source_type") - private int sourceType; - @JsonProperty("columns_stats") private List<ColumnStats> columnStats = new ArrayList<>(); @@ -66,6 +59,8 @@ public class TableExtDesc extends RootPersistentEntity implements ISourceAware { private String partitionColumn; @JsonProperty("total_file_size") private String totalFileSize; + @JsonProperty("total_rows") + private String totalRows; @JsonProperty("data_source_properties") private Map<String, String> dataSourceProps = new HashMap<>(); @@ -136,10 +131,6 @@ public class TableExtDesc extends RootPersistentEntity implements ISourceAware { } } - public void setSourceType(int sourceType) { - this.sourceType = sourceType; - } - public List<ColumnStats> getColumnStats() { return this.columnStats; } @@ -231,11 +222,6 @@ public class TableExtDesc extends RootPersistentEntity implements ISourceAware { return "TableExtDesc{" + "name='" + (null == tableName ? "NULL" : tableName) + '\'' + ", columns_samples=" + (null == columnStats ? "null" : Arrays.toString(columnStats.toArray())); } - @Override - public int getSourceType() { - return -1; - } - @JsonIgnoreProperties(ignoreUnknown = true) public static class ColumnStats implements Comparable<ColumnStats> { http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec4a4c/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java index d83ae28..3091325 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java @@ -19,20 +19,11 @@ package org.apache.kylin.rest.service; import java.io.IOException; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.engine.mr.CubingJob; -import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; -import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableManager; -import org.apache.kylin.job.execution.ExecutableState; -import org.apache.kylin.job.execution.Output; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.badquery.BadQueryHistoryManager; import org.apache.kylin.metadata.project.ProjectManager; @@ -40,12 +31,6 @@ import org.apache.kylin.metadata.streaming.StreamingManager; import org.apache.kylin.source.kafka.KafkaConfigManager; import org.apache.kylin.storage.hybrid.HybridManager; -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.Lists; - public abstract class BasicService { public KylinConfig getConfig() { @@ -94,59 +79,4 @@ public abstract class BasicService { return BadQueryHistoryManager.getInstance(getConfig()); } - protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) { - return listAllCubingJobs(cubeName, projectName, statusList, -1L, -1L, allOutputs); - } - - protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs) { - List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllExecutables(timeStartInMillis, timeEndInMillis)).filter(new Predicate<AbstractExecutable>() { - @Override - public boolean apply(AbstractExecutable executable) { - if (executable instanceof CubingJob) { - if (cubeName == null) { - return true; - } - return CubingExecutableUtil.getCubeName(executable.getParams()).equalsIgnoreCase(cubeName); - } else { - return false; - } - } - }).transform(new Function<AbstractExecutable, CubingJob>() { - @Override - public CubingJob apply(AbstractExecutable executable) { - return (CubingJob) executable; - } - }).filter(Predicates.and(new Predicate<CubingJob>() { - @Override - public boolean apply(CubingJob executable) { - if (null == projectName || null == getProjectManager().getProject(projectName)) { - return true; - } else { - return projectName.equals(executable.getParams().get("projectName")); - } - } - }, new Predicate<CubingJob>() { - @Override - public boolean apply(CubingJob executable) { - try { - Output output = allOutputs.get(executable.getId()); - ExecutableState state = output.getState(); - boolean ret = statusList.contains(state); - return ret; - } catch (Exception e) { - throw e; - } - } - }))); - return results; - } - - protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList) { - return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs()); - } - - protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName) { - return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), getExecutableManager().getAllOutputs()); - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec4a4c/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index c95f0ca..5252f59 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -94,6 +94,9 @@ public class CubeService extends BasicService { @Autowired private AccessService accessService; + @Autowired + private JobService jobService; + @PostFilter(Constant.ACCESS_POST_FILTER_READ) public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName) { List<CubeInstance> cubeInstances = null; @@ -228,7 +231,7 @@ public class CubeService extends BasicService { @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException, JobException { - final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); + final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); if (!cubingJobs.isEmpty()) { throw new JobException("Cube schema shouldn't be changed with running job."); } @@ -258,7 +261,7 @@ public class CubeService extends BasicService { @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") public void deleteCube(CubeInstance cube) throws IOException, JobException { - final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); + final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); if (!cubingJobs.isEmpty()) { throw new JobException("The cube " + cube.getName() + " has running job, please discard it and try again."); } @@ -358,7 +361,7 @@ public class CubeService extends BasicService { throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY segment"); } - final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); + final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); if (!cubingJobs.isEmpty()) { throw new JobException("Enable is not allowed with a running job."); } @@ -528,7 +531,7 @@ public class CubeService extends BasicService { } private void releaseAllJobs(CubeInstance cube) { - final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null); + final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null); for (CubingJob cubingJob : cubingJobs) { final ExecutableState status = cubingJob.getStatus(); if (status != ExecutableState.SUCCEED && status != ExecutableState.STOPPED && status != ExecutableState.DISCARDED) { http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec4a4c/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java index 7383c52..fa578fe 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -75,6 +75,8 @@ import org.springframework.stereotype.Component; import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.collect.FluentIterable; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -502,4 +504,61 @@ public class JobService extends BasicService implements InitializingBean { } return serverName; } + + public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) { + return listAllCubingJobs(cubeName, projectName, statusList, -1L, -1L, allOutputs); + } + + public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs) { + List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllExecutables(timeStartInMillis, timeEndInMillis)).filter(new Predicate<AbstractExecutable>() { + @Override + public boolean apply(AbstractExecutable executable) { + if (executable instanceof CubingJob) { + if (cubeName == null) { + return true; + } + return CubingExecutableUtil.getCubeName(executable.getParams()).equalsIgnoreCase(cubeName); + } else { + return false; + } + } + }).transform(new Function<AbstractExecutable, CubingJob>() { + @Override + public CubingJob apply(AbstractExecutable executable) { + return (CubingJob) executable; + } + }).filter(Predicates.and(new Predicate<CubingJob>() { + @Override + public boolean apply(CubingJob executable) { + if (null == projectName || null == getProjectManager().getProject(projectName)) { + return true; + } else { + return projectName.equals(executable.getProjectName()); + } + } + }, new Predicate<CubingJob>() { + @Override + public boolean apply(CubingJob executable) { + try { + Output output = allOutputs.get(executable.getId()); + ExecutableState state = output.getState(); + boolean ret = statusList.contains(state); + return ret; + } catch (Exception e) { + throw e; + } + } + }))); + return results; + } + + public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList) { + return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs()); + } + + public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName) { + return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), getExecutableManager().getAllOutputs()); + } + + }
