minor, add extract all projects command Signed-off-by: Hongbin Ma <mahong...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7510c27 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7510c27 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7510c27 Branch: refs/heads/master-hbase0.98 Commit: e7510c270642178ca208389dbfaf856c20f9a17d Parents: 8720362 Author: Roger Shi <rogershijich...@hotmail.com> Authored: Tue Mar 14 19:22:58 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Mar 14 19:30:33 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/tool/CubeMetaExtractor.java | 45 +++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e7510c27/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java index 35831f7..e370e48 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java @@ -23,8 +23,6 @@ import java.io.IOException; import java.util.List; import java.util.Set; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.OptionGroup; @@ -52,6 +50,7 @@ import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.project.RealizationEntry; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.RealizationRegistry; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.metadata.streaming.StreamingConfig; import org.apache.kylin.metadata.streaming.StreamingManager; @@ -61,8 +60,10 @@ import org.apache.kylin.storage.hybrid.HybridManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -80,6 +81,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify which hybrid to extract").create("hybrid"); @SuppressWarnings("static-access") private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project"); + @SuppressWarnings("static-access") + private static final Option OPTION_All_PROJECT = OptionBuilder.withArgName("allProjects").hasArg(false).isRequired(false).withDescription("Specify realizations in all projects to extract").create("allProjects"); @SuppressWarnings("static-access") private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageType").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageType"); @@ -130,6 +133,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { realizationOrProject.addOption(OPTION_CUBE); realizationOrProject.addOption(OPTION_PROJECT); realizationOrProject.addOption(OPTION_HYBRID); + realizationOrProject.addOption(OPTION_All_PROJECT); realizationOrProject.setRequired(true); options.addOptionGroup(realizationOrProject); @@ -160,23 +164,15 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { realizationRegistry = RealizationRegistry.getInstance(kylinConfig); badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig); - if (optionsHelper.hasOption(OPTION_PROJECT)) { + if (optionsHelper.hasOption(OPTION_All_PROJECT)) { + for (ProjectInstance projectInstance : projectManager.listAllProjects()) { + requireProject(projectInstance); + } + } else if (optionsHelper.hasOption(OPTION_PROJECT)) { String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT); for (String projectName : projectNames.split(",")) { ProjectInstance projectInstance = projectManager.getProject(projectName); - if (projectInstance == null) { - throw new IllegalArgumentException("Project " + projectName + " does not exist"); - } - addRequired(projectInstance.getResourcePath()); - List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries(); - for (RealizationEntry realizationEntry : realizationEntries) { - retrieveResourcePath(getRealization(realizationEntry)); - } - List<DataModelDesc> modelDescs = metadataManager.getModels(projectName); - for (DataModelDesc modelDesc : modelDescs) { - addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); - } - addOptional(badQueryHistoryManager.getBadQueriesForProject(projectName).getResourcePath()); + requireProject(projectInstance); } } else if (optionsHelper.hasOption(OPTION_CUBE)) { String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE); @@ -205,6 +201,22 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { engineOverwrite(new File(exportDir.getAbsolutePath())); } + private void requireProject(ProjectInstance projectInstance) throws IOException { + if (projectInstance == null) { + throw new IllegalArgumentException("Project " + projectInstance.getName() + " does not exist"); + } + addRequired(projectInstance.getResourcePath()); + List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries(); + for (RealizationEntry realizationEntry : realizationEntries) { + retrieveResourcePath(getRealization(realizationEntry)); + } + List<DataModelDesc> modelDescs = metadataManager.getModels(projectInstance.getName()); + for (DataModelDesc modelDesc : modelDescs) { + addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); + } + addOptional(badQueryHistoryManager.getBadQueriesForProject(projectInstance.getName()).getResourcePath()); + } + private void executeExtraction(String dest) { logger.info("The resource paths going to be extracted:"); for (String s : requiredResources) { @@ -359,6 +371,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { logger.warn("It's useless to set includeJobs to true when includeSegments is set to false"); } + cube.setStatus(RealizationStatusEnum.DISABLED); cubesToTrimAndSave.add(cube); } } else if (realization instanceof HybridInstance) {
