Repository: kylin Updated Branches: refs/heads/master ee520d698 -> 828a71fae
KYLIN-1546 Move to tool module Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/828a71fa Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/828a71fa Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/828a71fa Branch: refs/heads/master Commit: 828a71faeb37ff3db485130bbbe871a99a647cef Parents: ee520d6 Author: lidongsjtu <lid...@apache.org> Authored: Thu Mar 31 16:21:01 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Thu Mar 31 16:22:30 2016 +0800 ---------------------------------------------------------------------- assembly/pom.xml | 5 + .../apache/kylin/tool/CubeMetaExtractor.java | 340 ------------------- .../org/apache/kylin/tool/DiagnosisInfoCLI.java | 147 -------- .../org/apache/kylin/tool/JobInfoExtractor.java | 178 ---------- pom.xml | 1 + server/pom.xml | 6 + .../apache/kylin/rest/service/BasicService.java | 5 + .../kylin/rest/service/DiagnosisService.java | 45 +++ tool/pom.xml | 66 ++++ .../apache/kylin/tool/CubeMetaExtractor.java | 340 +++++++++++++++++++ .../org/apache/kylin/tool/DiagnosisInfoCLI.java | 147 ++++++++ .../org/apache/kylin/tool/JobInfoExtractor.java | 178 ++++++++++ 12 files changed, 793 insertions(+), 665 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml index 2dc8faa..3c42194 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -62,6 +62,11 @@ <artifactId>kylin-engine-streaming</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-tool</artifactId> + <version>${project.parent.version}</version> + </dependency> <dependency> <groupId>org.apache.kylin</groupId> http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/assembly/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java ---------------------------------------------------------------------- diff --git a/assembly/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/assembly/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java deleted file mode 100644 index 020f9ca..0000000 --- a/assembly/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.tool; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.OptionGroup; -import org.apache.commons.cli.Options; -import org.apache.commons.lang3.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.ResourceTool; -import org.apache.kylin.common.util.AbstractApplication; -import org.apache.kylin.common.util.OptionsHelper; -import org.apache.kylin.cube.CubeDescManager; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.engine.streaming.StreamingConfig; -import org.apache.kylin.engine.streaming.StreamingManager; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.job.dao.ExecutableDao; -import org.apache.kylin.job.dao.ExecutablePO; -import org.apache.kylin.job.exception.PersistentException; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.metadata.project.ProjectInstance; -import org.apache.kylin.metadata.project.ProjectManager; -import org.apache.kylin.metadata.project.RealizationEntry; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.RealizationRegistry; -import org.apache.kylin.metadata.realization.RealizationType; -import org.apache.kylin.source.kafka.KafkaConfigManager; -import org.apache.kylin.source.kafka.config.KafkaConfig; -import org.apache.kylin.storage.hybrid.HybridInstance; -import org.apache.kylin.storage.hybrid.HybridManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -/** - * extract cube related info for debugging/distributing purpose - * TODO: deal with II case - */ -public class CubeMetaExtractor extends AbstractApplication { - - private static final Logger logger = LoggerFactory.getLogger(CubeMetaExtractor.class); - - @SuppressWarnings("static-access") - private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube"); - @SuppressWarnings("static-access") - private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify which hybrid to extract").create("hybrid"); - @SuppressWarnings("static-access") - private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project"); - - @SuppressWarnings("static-access") - private static final Option OPTION_INCLUDE_SEGMENTS = OptionBuilder.withArgName("includeSegments").hasArg().isRequired(false).withDescription("set this to true if want extract the segments info. Default true").create("includeSegments"); - @SuppressWarnings("static-access") - private static final Option OPTION_INCLUDE_JOB = OptionBuilder.withArgName("includeJobs").hasArg().isRequired(false).withDescription("set this to true if want to extract job info/outputs too. Default false").create("includeJobs"); - @SuppressWarnings("static-access") - private static final Option OPTION_INCLUDE_SEGMENT_DETAILS = OptionBuilder.withArgName("includeSegmentDetails").hasArg().isRequired(false).withDescription("set this to true if want to extract segment details too, such as dict, tablesnapshot. Default false").create("includeSegmentDetails"); - - @SuppressWarnings("static-access") - private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(false).withDescription("specify the dest dir to save the related metadata").create("destDir"); - - private Options options = null; - private KylinConfig kylinConfig; - private MetadataManager metadataManager; - private ProjectManager projectManager; - private HybridManager hybridManager; - private CubeManager cubeManager; - private StreamingManager streamingManager; - private KafkaConfigManager kafkaConfigManager; - private CubeDescManager cubeDescManager; - private ExecutableDao executableDao; - private RealizationRegistry realizationRegistry; - - boolean includeSegments; - boolean includeJobs; - boolean includeSegmentDetails; - - List<String> requiredResources = Lists.newArrayList(); - List<String> optionalResources = Lists.newArrayList(); - List<CubeInstance> cubesToTrimAndSave = Lists.newArrayList();//these cubes needs to be saved skipping segments - - public CubeMetaExtractor() { - options = new Options(); - - OptionGroup realizationOrProject = new OptionGroup(); - realizationOrProject.addOption(OPTION_CUBE); - realizationOrProject.addOption(OPTION_PROJECT); - realizationOrProject.addOption(OPTION_HYBRID); - realizationOrProject.setRequired(true); - - options.addOptionGroup(realizationOrProject); - options.addOption(OPTION_INCLUDE_SEGMENTS); - options.addOption(OPTION_INCLUDE_JOB); - options.addOption(OPTION_INCLUDE_SEGMENT_DETAILS); - options.addOption(OPTION_DEST); - - } - - @Override - protected Options getOptions() { - return options; - } - - @Override - protected void execute(OptionsHelper optionsHelper) throws Exception { - includeSegments = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENTS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENTS)) : true; - includeJobs = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : false; - includeSegmentDetails = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENT_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENT_DETAILS)) : false; - - String dest = null; - if (optionsHelper.hasOption(OPTION_DEST)) { - dest = optionsHelper.getOptionValue(OPTION_DEST); - } - - if (StringUtils.isEmpty(dest)) { - throw new RuntimeException("destDir is not set, exit directly without extracting"); - } - - if (!dest.endsWith("/")) { - dest = dest + "/"; - } - - kylinConfig = KylinConfig.getInstanceFromEnv(); - metadataManager = MetadataManager.getInstance(kylinConfig); - projectManager = ProjectManager.getInstance(kylinConfig); - hybridManager = HybridManager.getInstance(kylinConfig); - cubeManager = CubeManager.getInstance(kylinConfig); - cubeDescManager = CubeDescManager.getInstance(kylinConfig); - streamingManager = StreamingManager.getInstance(kylinConfig); - kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig); - executableDao = ExecutableDao.getInstance(kylinConfig); - realizationRegistry = RealizationRegistry.getInstance(kylinConfig); - - if (optionsHelper.hasOption(OPTION_PROJECT)) { - ProjectInstance projectInstance = projectManager.getProject(optionsHelper.getOptionValue(OPTION_PROJECT)); - if (projectInstance == null) { - throw new IllegalArgumentException("Project " + optionsHelper.getOptionValue(OPTION_PROJECT) + " does not exist"); - } - addRequired(ProjectInstance.concatResourcePath(projectInstance.getName())); - List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries(); - for (RealizationEntry realizationEntry : realizationEntries) { - retrieveResourcePath(getRealization(realizationEntry)); - } - } else if (optionsHelper.hasOption(OPTION_CUBE)) { - String cubeName = optionsHelper.getOptionValue(OPTION_CUBE); - IRealization realization; - - if ((realization = cubeManager.getRealization(cubeName)) != null) { - retrieveResourcePath(realization); - } else { - throw new IllegalArgumentException("No cube found with name of " + cubeName); - } - } else if (optionsHelper.hasOption(OPTION_HYBRID)) { - String hybridName = optionsHelper.getOptionValue(OPTION_HYBRID); - IRealization realization; - - if ((realization = hybridManager.getRealization(hybridName)) != null) { - retrieveResourcePath(realization); - } else { - throw new IllegalArgumentException("No hybrid found with name of" + hybridName); - } - } - - executeExtraction(dest); - - logger.info("Extracted metadata files located at: " + new File(dest).getAbsolutePath()); - } - - private void executeExtraction(String dest) { - logger.info("The resource paths going to be extracted:"); - for (String s : requiredResources) { - logger.info(s + "(required)"); - } - for (String s : optionalResources) { - logger.info(s + "(optional)"); - } - for (CubeInstance cube : cubesToTrimAndSave) { - logger.info("Cube {} will be trimmed and extracted", cube); - } - - try { - ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); - ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest)); - - for (String path : requiredResources) { - ResourceTool.copyR(src, dst, path); - } - - for (String path : optionalResources) { - try { - ResourceTool.copyR(src, dst, path); - } catch (Exception e) { - logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it."); - } - } - - for (CubeInstance cube : cubesToTrimAndSave) { - CubeInstance trimmedCube = CubeInstance.getCopyOf(cube); - trimmedCube.getSegments().clear(); - trimmedCube.setUuid(cube.getUuid()); - dst.putResource(trimmedCube.getResourcePath(), trimmedCube, CubeManager.CUBE_SERIALIZER); - } - - } catch (IOException e) { - throw new RuntimeException("IOException", e); - } - } - - private IRealization getRealization(RealizationEntry realizationEntry) { - return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization()); - } - - private void dealWithStreaming(CubeInstance cube) { - for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) { - if (streamingConfig.getName() != null && streamingConfig.getName().equalsIgnoreCase(cube.getFactTable())) { - addRequired(StreamingConfig.concatResourcePath(streamingConfig.getName())); - addRequired(KafkaConfig.concatResourcePath(streamingConfig.getName())); - } - } - } - - private void retrieveResourcePath(IRealization realization) { - - logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType()); - - if (realization instanceof CubeInstance) { - CubeInstance cube = (CubeInstance) realization; - String descName = cube.getDescName(); - CubeDesc cubeDesc = cubeDescManager.getCubeDesc(descName); - String modelName = cubeDesc.getModelName(); - DataModelDesc modelDesc = metadataManager.getDataModelDesc(modelName); - - dealWithStreaming(cube); - - for (String tableName : modelDesc.getAllTables()) { - addRequired(TableDesc.concatResourcePath(tableName)); - addOptional(TableDesc.concatExdResourcePath(tableName)); - } - - addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); - addRequired(CubeDesc.concatResourcePath(cubeDesc.getName())); - - if (includeSegments) { - addRequired(CubeInstance.concatResourcePath(cube.getName())); - for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) { - if (includeSegmentDetails) { - for (String dictPat : segment.getDictionaryPaths()) { - addRequired(dictPat); - } - for (String snapshotPath : segment.getSnapshotPaths()) { - addRequired(snapshotPath); - } - addRequired(segment.getStatisticsResourcePath()); - } - - if (includeJobs) { - String lastJobId = segment.getLastBuildJobID(); - if (StringUtils.isEmpty(lastJobId)) { - throw new RuntimeException("No job exist for segment :" + segment); - } else { - try { - ExecutablePO executablePO = executableDao.getJob(lastJobId); - addRequired(ExecutableDao.pathOfJob(lastJobId)); - addRequired(ExecutableDao.pathOfJobOutput(lastJobId)); - for (ExecutablePO task : executablePO.getTasks()) { - addRequired(ExecutableDao.pathOfJob(task.getUuid())); - addRequired(ExecutableDao.pathOfJobOutput(task.getUuid())); - } - } catch (PersistentException e) { - throw new RuntimeException("PersistentException", e); - } - } - } - } - } else { - if (includeJobs) { - logger.warn("It's useless to set includeJobs to true when includeSegments is set to false"); - } - - cubesToTrimAndSave.add(cube); - } - } else if (realization instanceof HybridInstance) { - HybridInstance hybridInstance = (HybridInstance) realization; - addRequired(HybridInstance.concatResourcePath(hybridInstance.getName())); - for (IRealization iRealization : hybridInstance.getRealizations()) { - if (iRealization.getType() != RealizationType.CUBE) { - throw new RuntimeException("Hybrid " + iRealization.getName() + " contains non cube child " + iRealization.getName() + " with type " + iRealization.getType()); - } - retrieveResourcePath(iRealization); - } - } else if (realization instanceof IIInstance) { - throw new IllegalStateException("Does not support extract II instance or hybrid that contains II"); - } else { - throw new IllegalStateException("Unknown realization type: " + realization.getType()); - } - } - - private void addRequired(String record) { - logger.info("adding required resource {}", record); - requiredResources.add(record); - } - - private void addOptional(String record) { - logger.info("adding optional resource {}", record); - optionalResources.add(record); - } - - public static void main(String[] args) { - CubeMetaExtractor extractor = new CubeMetaExtractor(); - extractor.execute(args); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/assembly/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java ---------------------------------------------------------------------- diff --git a/assembly/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/assembly/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java deleted file mode 100644 index 62379f2..0000000 --- a/assembly/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.tool; - -import java.io.File; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; - -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.AbstractApplication; -import org.apache.kylin.common.util.OptionsHelper; -import org.apache.kylin.common.util.ZipFileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -public class DiagnosisInfoCLI extends AbstractApplication { - private static final Logger logger = LoggerFactory.getLogger(DiagnosisInfoCLI.class); - - private static final int DEFAULT_LOG_PERIOD = 3; - - @SuppressWarnings("static-access") - private static final Option OPTION_LOG_PERIOD = OptionBuilder.withArgName("logPeriod").hasArg().isRequired(false).withDescription("specify how many days of kylin logs to extract. Default 3.").create("logPeriod"); - - @SuppressWarnings("static-access") - private static final Option OPTION_COMPRESS = OptionBuilder.withArgName("compress").hasArg().isRequired(false).withDescription("specify whether to compress the output with zip. Default false.").create("compress"); - - private CubeMetaExtractor cubeMetaExtractor; - private JobInfoExtractor jobInfoExtractor; - private Options options; - private String type; - private String exportDest; - - public DiagnosisInfoCLI(String type) { - this.type = type; - - jobInfoExtractor = new JobInfoExtractor(); - cubeMetaExtractor = new CubeMetaExtractor(); - - if (this.type.equalsIgnoreCase("job")) { - options = jobInfoExtractor.getOptions(); - } else if (this.type.equalsIgnoreCase("metadata")) { - options = cubeMetaExtractor.getOptions(); - } else { - throw new RuntimeException("Only job and metadata are allowed."); - } - - options.addOption(OPTION_LOG_PERIOD); - options.addOption(OPTION_COMPRESS); - } - - public static void main(String args[]) { - DiagnosisInfoCLI diagnosisInfoCLI = new DiagnosisInfoCLI(args[0]); - diagnosisInfoCLI.execute(Arrays.copyOfRange(args, 1, args.length)); - } - - @Override - protected Options getOptions() { - return options; - } - - @Override - protected void execute(OptionsHelper optionsHelper) throws Exception { - - if (this.type.equals("job")) { - jobInfoExtractor.execute(optionsHelper); - exportDest = optionsHelper.getOptionValue(options.getOption("destDir")); - } else if (this.type.equals("metadata")) { - cubeMetaExtractor.execute(optionsHelper); - exportDest = optionsHelper.getOptionValue(options.getOption("destDir")); - } - - if (StringUtils.isEmpty(exportDest)) { - throw new RuntimeException("destDir is not set, exit directly without extracting"); - } - if (!exportDest.endsWith("/")) { - exportDest = exportDest + "/"; - } - - int logPeriod = optionsHelper.hasOption(OPTION_LOG_PERIOD) ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_LOG_PERIOD)) : DEFAULT_LOG_PERIOD; - boolean compress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : false; - - if (logPeriod > 0) { - logger.info("Start to extract kylin logs in {} days", logPeriod); - - final String logFolder = KylinConfig.getKylinHome() + "/logs/"; - final String defaultLogFilename = "kylin.log"; - final File logsDir = new File(exportDest + "/logs/"); - final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - - FileUtils.forceMkdir(logsDir); - - final ArrayList<String> logFileNames = Lists.newArrayListWithCapacity(logPeriod); - - logFileNames.add(defaultLogFilename); - for (int i = 1; i < logPeriod; i++) { - Calendar todayCal = Calendar.getInstance(); - todayCal.add(Calendar.DAY_OF_MONTH, 0 - i); - logFileNames.add(defaultLogFilename + "." + format.format(todayCal.getTime())); - } - - for (String logFilename : logFileNames) { - File logFile = new File(logFolder + logFilename); - if (logFile.exists()) { - FileUtils.copyFileToDirectory(logFile, logsDir); - } - } - } - - if (compress) { - File tempZipFile = File.createTempFile("diagnosis_", ".zip"); - ZipFileUtils.compressZipFile(exportDest, tempZipFile.getAbsolutePath()); - FileUtils.forceDelete(new File(exportDest)); - FileUtils.moveFileToDirectory(tempZipFile, new File(exportDest), true); - exportDest = exportDest + tempZipFile.getName(); - } - logger.info("Diagnosis info locates at: " + new File(exportDest).getAbsolutePath()); - } - - public String getExportDest() { - return exportDest; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/assembly/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java ---------------------------------------------------------------------- diff --git a/assembly/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java b/assembly/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java deleted file mode 100644 index 43758e0..0000000 --- a/assembly/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.tool; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.ResourceTool; -import org.apache.kylin.common.util.AbstractApplication; -import org.apache.kylin.common.util.OptionsHelper; -import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; -import org.apache.kylin.job.common.ShellExecutable; -import org.apache.kylin.job.constant.ExecutableConstants; -import org.apache.kylin.job.dao.ExecutableDao; -import org.apache.kylin.job.dao.ExecutablePO; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -/** - * Created by dongli on 3/29/16. - */ -public class JobInfoExtractor extends AbstractApplication { - private static final Logger logger = LoggerFactory.getLogger(JobInfoExtractor.class); - - @SuppressWarnings("static-access") - private static final Option OPTION_JOB_ID = OptionBuilder.withArgName("jobId").hasArg().isRequired(true).withDescription("specify the Job ID to extract information. ").create("jobId"); - - @SuppressWarnings("static-access") - private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(true).withDescription("specify the dest dir to save the related information").create("destDir"); - - @SuppressWarnings("static-access") - private static final Option OPTION_INCLUDE_CUBE = OptionBuilder.withArgName("includeCube").hasArg().isRequired(false).withDescription("set this to true if want to extract related cube info too. Default true").create("includeCube"); - - @SuppressWarnings("static-access") - private static final Option OPTION_INCLUDE_YARN_LOGS = OptionBuilder.withArgName("includeYarnLogs").hasArg().isRequired(false).withDescription("set this to true if want to extract related yarn logs too. Default true").create("includeYarnLogs"); - - private Options options; - - private KylinConfig kylinConfig; - private CubeMetaExtractor cubeMetaExtractor; - - private ExecutableDao executableDao; - - List<String> requiredResources = Lists.newArrayList(); - List<String> yarnLogsResources = Lists.newArrayList(); - - public JobInfoExtractor() { - cubeMetaExtractor = new CubeMetaExtractor(); - - options = new Options(); - options.addOption(OPTION_JOB_ID); - options.addOption(OPTION_DEST); - options.addOption(OPTION_INCLUDE_CUBE); - options.addOption(OPTION_INCLUDE_YARN_LOGS); - - kylinConfig = KylinConfig.getInstanceFromEnv(); - executableDao = ExecutableDao.getInstance(kylinConfig); - } - - @Override - protected Options getOptions() { - return options; - } - - @Override - protected void execute(OptionsHelper optionsHelper) throws Exception { - String jobId = optionsHelper.getOptionValue(OPTION_JOB_ID); - String dest = optionsHelper.getOptionValue(OPTION_DEST); - boolean includeCube = optionsHelper.hasOption(OPTION_INCLUDE_CUBE) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CUBE)) : true; - boolean includeYarnLogs = optionsHelper.hasOption(OPTION_INCLUDE_YARN_LOGS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_YARN_LOGS)) : true; - - if (StringUtils.isEmpty(dest)) { - throw new RuntimeException("destDir is not set, exit directly without extracting"); - } - - if (!dest.endsWith("/")) { - dest = dest + "/"; - } - - ExecutablePO executablePO = executableDao.getJob(jobId); - addRequired(ExecutableDao.pathOfJob(jobId)); - addRequired(ExecutableDao.pathOfJobOutput(jobId)); - for (ExecutablePO task : executablePO.getTasks()) { - addRequired(ExecutableDao.pathOfJob(task.getUuid())); - addRequired(ExecutableDao.pathOfJobOutput(task.getUuid())); - if (includeYarnLogs) { - yarnLogsResources.add(task.getUuid()); - } - } - executeExtraction(dest); - - if (includeCube) { - String cubeName = CubingExecutableUtil.getCubeName(executablePO.getParams()); - String[] cubeMetaArgs = { "-cube", cubeName, "-destDir", dest + "cube_" + cubeName + "/", "-includeJobs", "false" }; - logger.info("Start to extract related cube: " + StringUtils.join(cubeMetaArgs)); - cubeMetaExtractor.execute(cubeMetaArgs); - } - - if (includeYarnLogs) { - logger.info("Start to related yarn job logs: " + jobId); - for (String taskId : yarnLogsResources) { - extractYarnLog(taskId, dest + "yarn_" + jobId + "/"); - } - } - - logger.info("Extracted kylin jobs located at: " + new File(dest).getAbsolutePath()); - } - - private void executeExtraction(String dest) { - logger.info("The resource paths going to be extracted:"); - for (String s : requiredResources) { - logger.info(s + "(required)"); - } - - try { - ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); - ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest)); - - for (String path : requiredResources) { - ResourceTool.copyR(src, dst, path); - } - - } catch (IOException e) { - throw new RuntimeException("IOException", e); - } - } - - private void extractYarnLog(String taskId, String dest) throws Exception { - final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo(); - if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) { - String applicationId = jobInfo.get(ExecutableConstants.MR_JOB_ID).replace("job", "application"); - File destFile = new File(dest + applicationId + ".log"); - - ShellExecutable yarnExec = new ShellExecutable(); - yarnExec.setCmd("yarn logs -applicationId " + applicationId + " > " + destFile.getAbsolutePath()); - yarnExec.setName(yarnExec.getCmd()); - - logger.info(yarnExec.getCmd()); - kylinConfig.getCliCommandExecutor().execute(yarnExec.getCmd(), null); - } - } - - private void addRequired(String record) { - logger.info("adding required resource {}", record); - requiredResources.add(record); - } - - public static void main(String args[]) { - JobInfoExtractor extractor = new JobInfoExtractor(); - extractor.execute(args); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ac87fd3..27bb1cb 100644 --- a/pom.xml +++ b/pom.xml @@ -629,6 +629,7 @@ <module>jdbc</module> <module>invertedindex</module> <module>assembly</module> + <module>tool</module> <module>kylin-it</module> </modules> http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/server/pom.xml ---------------------------------------------------------------------- diff --git a/server/pom.xml b/server/pom.xml index b804933..49f18c3 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -68,6 +68,12 @@ <artifactId>kylin-source-kafka</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-tool</artifactId> + <version>${project.parent.version}</version> + </dependency> + <!-- Test & Env --> <dependency> <groupId>org.apache.kylin</groupId> http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/server/src/main/java/org/apache/kylin/rest/service/BasicService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java index 7696c59..7197f03 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java @@ -37,6 +37,7 @@ import org.apache.kylin.job.execution.ExecutableState; import org.apache.kylin.job.execution.Output; import org.apache.kylin.job.manager.ExecutableManager; import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.badquery.BadQueryHistoryManager; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.RealizationType; @@ -101,6 +102,10 @@ public abstract class BasicService { return IIManager.getInstance(getConfig()); } + public BadQueryHistoryManager getBadQueryHistoryManager() { + return BadQueryHistoryManager.getInstance(getConfig()); + } + protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) { return listAllCubingJobs(cubeName, projectName, statusList, -1L, -1L, allOutputs); } http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java b/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java new file mode 100644 index 0000000..e7d6cd1 --- /dev/null +++ b/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.service; + +import java.io.IOException; + +import org.apache.kylin.metadata.badquery.BadQueryHistory; +import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.tool.DiagnosisInfoCLI; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.stereotype.Component; + +@Component("diagnosisService") +public class DiagnosisService extends BasicService { + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) + public BadQueryHistory getProjectBadQueryHistory(String project) throws IOException { + return getBadQueryHistoryManager().getBadQueriesForProject(project); + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) + public String dumpDiagnosisInfo(String project) throws IOException { + String tempLocation = System.getProperty("java.io.tmpdir"); + String[] args = { "-project", project, "-destDir", tempLocation, "-compress", "true" }; + DiagnosisInfoCLI diagnosisInfoCli = new DiagnosisInfoCLI("metadata"); + diagnosisInfoCli.execute(args); + return diagnosisInfoCli.getExportDest(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml new file mode 100644 index 0000000..2cc9a7e --- /dev/null +++ b/tool/pom.xml @@ -0,0 +1,66 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>kylin</artifactId> + <groupId>org.apache.kylin</groupId> + <version>1.5.1-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>kylin-tool</artifactId> + <name>Kylin:Tool</name> + + <dependencies> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-source-hive</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-source-kafka</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-storage-hbase</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-engine-mr</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-engine-streaming</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-invertedindex</artifactId> + <version>${project.parent.version}</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java new file mode 100644 index 0000000..020f9ca --- /dev/null +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.tool; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.ResourceTool; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.OptionsHelper; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.engine.streaming.StreamingConfig; +import org.apache.kylin.engine.streaming.StreamingManager; +import org.apache.kylin.invertedindex.IIInstance; +import org.apache.kylin.job.dao.ExecutableDao; +import org.apache.kylin.job.dao.ExecutablePO; +import org.apache.kylin.job.exception.PersistentException; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.SegmentStatusEnum; +import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.project.RealizationEntry; +import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.metadata.realization.RealizationRegistry; +import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.source.kafka.KafkaConfigManager; +import org.apache.kylin.source.kafka.config.KafkaConfig; +import org.apache.kylin.storage.hybrid.HybridInstance; +import org.apache.kylin.storage.hybrid.HybridManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +/** + * extract cube related info for debugging/distributing purpose + * TODO: deal with II case + */ +public class CubeMetaExtractor extends AbstractApplication { + + private static final Logger logger = LoggerFactory.getLogger(CubeMetaExtractor.class); + + @SuppressWarnings("static-access") + private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube"); + @SuppressWarnings("static-access") + private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify which hybrid to extract").create("hybrid"); + @SuppressWarnings("static-access") + private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project"); + + @SuppressWarnings("static-access") + private static final Option OPTION_INCLUDE_SEGMENTS = OptionBuilder.withArgName("includeSegments").hasArg().isRequired(false).withDescription("set this to true if want extract the segments info. Default true").create("includeSegments"); + @SuppressWarnings("static-access") + private static final Option OPTION_INCLUDE_JOB = OptionBuilder.withArgName("includeJobs").hasArg().isRequired(false).withDescription("set this to true if want to extract job info/outputs too. Default false").create("includeJobs"); + @SuppressWarnings("static-access") + private static final Option OPTION_INCLUDE_SEGMENT_DETAILS = OptionBuilder.withArgName("includeSegmentDetails").hasArg().isRequired(false).withDescription("set this to true if want to extract segment details too, such as dict, tablesnapshot. Default false").create("includeSegmentDetails"); + + @SuppressWarnings("static-access") + private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(false).withDescription("specify the dest dir to save the related metadata").create("destDir"); + + private Options options = null; + private KylinConfig kylinConfig; + private MetadataManager metadataManager; + private ProjectManager projectManager; + private HybridManager hybridManager; + private CubeManager cubeManager; + private StreamingManager streamingManager; + private KafkaConfigManager kafkaConfigManager; + private CubeDescManager cubeDescManager; + private ExecutableDao executableDao; + private RealizationRegistry realizationRegistry; + + boolean includeSegments; + boolean includeJobs; + boolean includeSegmentDetails; + + List<String> requiredResources = Lists.newArrayList(); + List<String> optionalResources = Lists.newArrayList(); + List<CubeInstance> cubesToTrimAndSave = Lists.newArrayList();//these cubes needs to be saved skipping segments + + public CubeMetaExtractor() { + options = new Options(); + + OptionGroup realizationOrProject = new OptionGroup(); + realizationOrProject.addOption(OPTION_CUBE); + realizationOrProject.addOption(OPTION_PROJECT); + realizationOrProject.addOption(OPTION_HYBRID); + realizationOrProject.setRequired(true); + + options.addOptionGroup(realizationOrProject); + options.addOption(OPTION_INCLUDE_SEGMENTS); + options.addOption(OPTION_INCLUDE_JOB); + options.addOption(OPTION_INCLUDE_SEGMENT_DETAILS); + options.addOption(OPTION_DEST); + + } + + @Override + protected Options getOptions() { + return options; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + includeSegments = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENTS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENTS)) : true; + includeJobs = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : false; + includeSegmentDetails = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENT_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENT_DETAILS)) : false; + + String dest = null; + if (optionsHelper.hasOption(OPTION_DEST)) { + dest = optionsHelper.getOptionValue(OPTION_DEST); + } + + if (StringUtils.isEmpty(dest)) { + throw new RuntimeException("destDir is not set, exit directly without extracting"); + } + + if (!dest.endsWith("/")) { + dest = dest + "/"; + } + + kylinConfig = KylinConfig.getInstanceFromEnv(); + metadataManager = MetadataManager.getInstance(kylinConfig); + projectManager = ProjectManager.getInstance(kylinConfig); + hybridManager = HybridManager.getInstance(kylinConfig); + cubeManager = CubeManager.getInstance(kylinConfig); + cubeDescManager = CubeDescManager.getInstance(kylinConfig); + streamingManager = StreamingManager.getInstance(kylinConfig); + kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig); + executableDao = ExecutableDao.getInstance(kylinConfig); + realizationRegistry = RealizationRegistry.getInstance(kylinConfig); + + if (optionsHelper.hasOption(OPTION_PROJECT)) { + ProjectInstance projectInstance = projectManager.getProject(optionsHelper.getOptionValue(OPTION_PROJECT)); + if (projectInstance == null) { + throw new IllegalArgumentException("Project " + optionsHelper.getOptionValue(OPTION_PROJECT) + " does not exist"); + } + addRequired(ProjectInstance.concatResourcePath(projectInstance.getName())); + List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries(); + for (RealizationEntry realizationEntry : realizationEntries) { + retrieveResourcePath(getRealization(realizationEntry)); + } + } else if (optionsHelper.hasOption(OPTION_CUBE)) { + String cubeName = optionsHelper.getOptionValue(OPTION_CUBE); + IRealization realization; + + if ((realization = cubeManager.getRealization(cubeName)) != null) { + retrieveResourcePath(realization); + } else { + throw new IllegalArgumentException("No cube found with name of " + cubeName); + } + } else if (optionsHelper.hasOption(OPTION_HYBRID)) { + String hybridName = optionsHelper.getOptionValue(OPTION_HYBRID); + IRealization realization; + + if ((realization = hybridManager.getRealization(hybridName)) != null) { + retrieveResourcePath(realization); + } else { + throw new IllegalArgumentException("No hybrid found with name of" + hybridName); + } + } + + executeExtraction(dest); + + logger.info("Extracted metadata files located at: " + new File(dest).getAbsolutePath()); + } + + private void executeExtraction(String dest) { + logger.info("The resource paths going to be extracted:"); + for (String s : requiredResources) { + logger.info(s + "(required)"); + } + for (String s : optionalResources) { + logger.info(s + "(optional)"); + } + for (CubeInstance cube : cubesToTrimAndSave) { + logger.info("Cube {} will be trimmed and extracted", cube); + } + + try { + ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); + ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest)); + + for (String path : requiredResources) { + ResourceTool.copyR(src, dst, path); + } + + for (String path : optionalResources) { + try { + ResourceTool.copyR(src, dst, path); + } catch (Exception e) { + logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it."); + } + } + + for (CubeInstance cube : cubesToTrimAndSave) { + CubeInstance trimmedCube = CubeInstance.getCopyOf(cube); + trimmedCube.getSegments().clear(); + trimmedCube.setUuid(cube.getUuid()); + dst.putResource(trimmedCube.getResourcePath(), trimmedCube, CubeManager.CUBE_SERIALIZER); + } + + } catch (IOException e) { + throw new RuntimeException("IOException", e); + } + } + + private IRealization getRealization(RealizationEntry realizationEntry) { + return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization()); + } + + private void dealWithStreaming(CubeInstance cube) { + for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) { + if (streamingConfig.getName() != null && streamingConfig.getName().equalsIgnoreCase(cube.getFactTable())) { + addRequired(StreamingConfig.concatResourcePath(streamingConfig.getName())); + addRequired(KafkaConfig.concatResourcePath(streamingConfig.getName())); + } + } + } + + private void retrieveResourcePath(IRealization realization) { + + logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType()); + + if (realization instanceof CubeInstance) { + CubeInstance cube = (CubeInstance) realization; + String descName = cube.getDescName(); + CubeDesc cubeDesc = cubeDescManager.getCubeDesc(descName); + String modelName = cubeDesc.getModelName(); + DataModelDesc modelDesc = metadataManager.getDataModelDesc(modelName); + + dealWithStreaming(cube); + + for (String tableName : modelDesc.getAllTables()) { + addRequired(TableDesc.concatResourcePath(tableName)); + addOptional(TableDesc.concatExdResourcePath(tableName)); + } + + addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); + addRequired(CubeDesc.concatResourcePath(cubeDesc.getName())); + + if (includeSegments) { + addRequired(CubeInstance.concatResourcePath(cube.getName())); + for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) { + if (includeSegmentDetails) { + for (String dictPat : segment.getDictionaryPaths()) { + addRequired(dictPat); + } + for (String snapshotPath : segment.getSnapshotPaths()) { + addRequired(snapshotPath); + } + addRequired(segment.getStatisticsResourcePath()); + } + + if (includeJobs) { + String lastJobId = segment.getLastBuildJobID(); + if (StringUtils.isEmpty(lastJobId)) { + throw new RuntimeException("No job exist for segment :" + segment); + } else { + try { + ExecutablePO executablePO = executableDao.getJob(lastJobId); + addRequired(ExecutableDao.pathOfJob(lastJobId)); + addRequired(ExecutableDao.pathOfJobOutput(lastJobId)); + for (ExecutablePO task : executablePO.getTasks()) { + addRequired(ExecutableDao.pathOfJob(task.getUuid())); + addRequired(ExecutableDao.pathOfJobOutput(task.getUuid())); + } + } catch (PersistentException e) { + throw new RuntimeException("PersistentException", e); + } + } + } + } + } else { + if (includeJobs) { + logger.warn("It's useless to set includeJobs to true when includeSegments is set to false"); + } + + cubesToTrimAndSave.add(cube); + } + } else if (realization instanceof HybridInstance) { + HybridInstance hybridInstance = (HybridInstance) realization; + addRequired(HybridInstance.concatResourcePath(hybridInstance.getName())); + for (IRealization iRealization : hybridInstance.getRealizations()) { + if (iRealization.getType() != RealizationType.CUBE) { + throw new RuntimeException("Hybrid " + iRealization.getName() + " contains non cube child " + iRealization.getName() + " with type " + iRealization.getType()); + } + retrieveResourcePath(iRealization); + } + } else if (realization instanceof IIInstance) { + throw new IllegalStateException("Does not support extract II instance or hybrid that contains II"); + } else { + throw new IllegalStateException("Unknown realization type: " + realization.getType()); + } + } + + private void addRequired(String record) { + logger.info("adding required resource {}", record); + requiredResources.add(record); + } + + private void addOptional(String record) { + logger.info("adding optional resource {}", record); + optionalResources.add(record); + } + + public static void main(String[] args) { + CubeMetaExtractor extractor = new CubeMetaExtractor(); + extractor.execute(args); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java new file mode 100644 index 0000000..62379f2 --- /dev/null +++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.tool; + +import java.io.File; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.OptionsHelper; +import org.apache.kylin.common.util.ZipFileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class DiagnosisInfoCLI extends AbstractApplication { + private static final Logger logger = LoggerFactory.getLogger(DiagnosisInfoCLI.class); + + private static final int DEFAULT_LOG_PERIOD = 3; + + @SuppressWarnings("static-access") + private static final Option OPTION_LOG_PERIOD = OptionBuilder.withArgName("logPeriod").hasArg().isRequired(false).withDescription("specify how many days of kylin logs to extract. Default 3.").create("logPeriod"); + + @SuppressWarnings("static-access") + private static final Option OPTION_COMPRESS = OptionBuilder.withArgName("compress").hasArg().isRequired(false).withDescription("specify whether to compress the output with zip. Default false.").create("compress"); + + private CubeMetaExtractor cubeMetaExtractor; + private JobInfoExtractor jobInfoExtractor; + private Options options; + private String type; + private String exportDest; + + public DiagnosisInfoCLI(String type) { + this.type = type; + + jobInfoExtractor = new JobInfoExtractor(); + cubeMetaExtractor = new CubeMetaExtractor(); + + if (this.type.equalsIgnoreCase("job")) { + options = jobInfoExtractor.getOptions(); + } else if (this.type.equalsIgnoreCase("metadata")) { + options = cubeMetaExtractor.getOptions(); + } else { + throw new RuntimeException("Only job and metadata are allowed."); + } + + options.addOption(OPTION_LOG_PERIOD); + options.addOption(OPTION_COMPRESS); + } + + public static void main(String args[]) { + DiagnosisInfoCLI diagnosisInfoCLI = new DiagnosisInfoCLI(args[0]); + diagnosisInfoCLI.execute(Arrays.copyOfRange(args, 1, args.length)); + } + + @Override + protected Options getOptions() { + return options; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + + if (this.type.equals("job")) { + jobInfoExtractor.execute(optionsHelper); + exportDest = optionsHelper.getOptionValue(options.getOption("destDir")); + } else if (this.type.equals("metadata")) { + cubeMetaExtractor.execute(optionsHelper); + exportDest = optionsHelper.getOptionValue(options.getOption("destDir")); + } + + if (StringUtils.isEmpty(exportDest)) { + throw new RuntimeException("destDir is not set, exit directly without extracting"); + } + if (!exportDest.endsWith("/")) { + exportDest = exportDest + "/"; + } + + int logPeriod = optionsHelper.hasOption(OPTION_LOG_PERIOD) ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_LOG_PERIOD)) : DEFAULT_LOG_PERIOD; + boolean compress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : false; + + if (logPeriod > 0) { + logger.info("Start to extract kylin logs in {} days", logPeriod); + + final String logFolder = KylinConfig.getKylinHome() + "/logs/"; + final String defaultLogFilename = "kylin.log"; + final File logsDir = new File(exportDest + "/logs/"); + final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + + FileUtils.forceMkdir(logsDir); + + final ArrayList<String> logFileNames = Lists.newArrayListWithCapacity(logPeriod); + + logFileNames.add(defaultLogFilename); + for (int i = 1; i < logPeriod; i++) { + Calendar todayCal = Calendar.getInstance(); + todayCal.add(Calendar.DAY_OF_MONTH, 0 - i); + logFileNames.add(defaultLogFilename + "." + format.format(todayCal.getTime())); + } + + for (String logFilename : logFileNames) { + File logFile = new File(logFolder + logFilename); + if (logFile.exists()) { + FileUtils.copyFileToDirectory(logFile, logsDir); + } + } + } + + if (compress) { + File tempZipFile = File.createTempFile("diagnosis_", ".zip"); + ZipFileUtils.compressZipFile(exportDest, tempZipFile.getAbsolutePath()); + FileUtils.forceDelete(new File(exportDest)); + FileUtils.moveFileToDirectory(tempZipFile, new File(exportDest), true); + exportDest = exportDest + tempZipFile.getName(); + } + logger.info("Diagnosis info locates at: " + new File(exportDest).getAbsolutePath()); + } + + public String getExportDest() { + return exportDest; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/828a71fa/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java new file mode 100644 index 0000000..43758e0 --- /dev/null +++ b/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.tool; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.ResourceTool; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.OptionsHelper; +import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; +import org.apache.kylin.job.common.ShellExecutable; +import org.apache.kylin.job.constant.ExecutableConstants; +import org.apache.kylin.job.dao.ExecutableDao; +import org.apache.kylin.job.dao.ExecutablePO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +/** + * Created by dongli on 3/29/16. + */ +public class JobInfoExtractor extends AbstractApplication { + private static final Logger logger = LoggerFactory.getLogger(JobInfoExtractor.class); + + @SuppressWarnings("static-access") + private static final Option OPTION_JOB_ID = OptionBuilder.withArgName("jobId").hasArg().isRequired(true).withDescription("specify the Job ID to extract information. ").create("jobId"); + + @SuppressWarnings("static-access") + private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(true).withDescription("specify the dest dir to save the related information").create("destDir"); + + @SuppressWarnings("static-access") + private static final Option OPTION_INCLUDE_CUBE = OptionBuilder.withArgName("includeCube").hasArg().isRequired(false).withDescription("set this to true if want to extract related cube info too. Default true").create("includeCube"); + + @SuppressWarnings("static-access") + private static final Option OPTION_INCLUDE_YARN_LOGS = OptionBuilder.withArgName("includeYarnLogs").hasArg().isRequired(false).withDescription("set this to true if want to extract related yarn logs too. Default true").create("includeYarnLogs"); + + private Options options; + + private KylinConfig kylinConfig; + private CubeMetaExtractor cubeMetaExtractor; + + private ExecutableDao executableDao; + + List<String> requiredResources = Lists.newArrayList(); + List<String> yarnLogsResources = Lists.newArrayList(); + + public JobInfoExtractor() { + cubeMetaExtractor = new CubeMetaExtractor(); + + options = new Options(); + options.addOption(OPTION_JOB_ID); + options.addOption(OPTION_DEST); + options.addOption(OPTION_INCLUDE_CUBE); + options.addOption(OPTION_INCLUDE_YARN_LOGS); + + kylinConfig = KylinConfig.getInstanceFromEnv(); + executableDao = ExecutableDao.getInstance(kylinConfig); + } + + @Override + protected Options getOptions() { + return options; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + String jobId = optionsHelper.getOptionValue(OPTION_JOB_ID); + String dest = optionsHelper.getOptionValue(OPTION_DEST); + boolean includeCube = optionsHelper.hasOption(OPTION_INCLUDE_CUBE) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CUBE)) : true; + boolean includeYarnLogs = optionsHelper.hasOption(OPTION_INCLUDE_YARN_LOGS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_YARN_LOGS)) : true; + + if (StringUtils.isEmpty(dest)) { + throw new RuntimeException("destDir is not set, exit directly without extracting"); + } + + if (!dest.endsWith("/")) { + dest = dest + "/"; + } + + ExecutablePO executablePO = executableDao.getJob(jobId); + addRequired(ExecutableDao.pathOfJob(jobId)); + addRequired(ExecutableDao.pathOfJobOutput(jobId)); + for (ExecutablePO task : executablePO.getTasks()) { + addRequired(ExecutableDao.pathOfJob(task.getUuid())); + addRequired(ExecutableDao.pathOfJobOutput(task.getUuid())); + if (includeYarnLogs) { + yarnLogsResources.add(task.getUuid()); + } + } + executeExtraction(dest); + + if (includeCube) { + String cubeName = CubingExecutableUtil.getCubeName(executablePO.getParams()); + String[] cubeMetaArgs = { "-cube", cubeName, "-destDir", dest + "cube_" + cubeName + "/", "-includeJobs", "false" }; + logger.info("Start to extract related cube: " + StringUtils.join(cubeMetaArgs)); + cubeMetaExtractor.execute(cubeMetaArgs); + } + + if (includeYarnLogs) { + logger.info("Start to related yarn job logs: " + jobId); + for (String taskId : yarnLogsResources) { + extractYarnLog(taskId, dest + "yarn_" + jobId + "/"); + } + } + + logger.info("Extracted kylin jobs located at: " + new File(dest).getAbsolutePath()); + } + + private void executeExtraction(String dest) { + logger.info("The resource paths going to be extracted:"); + for (String s : requiredResources) { + logger.info(s + "(required)"); + } + + try { + ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); + ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest)); + + for (String path : requiredResources) { + ResourceTool.copyR(src, dst, path); + } + + } catch (IOException e) { + throw new RuntimeException("IOException", e); + } + } + + private void extractYarnLog(String taskId, String dest) throws Exception { + final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo(); + if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) { + String applicationId = jobInfo.get(ExecutableConstants.MR_JOB_ID).replace("job", "application"); + File destFile = new File(dest + applicationId + ".log"); + + ShellExecutable yarnExec = new ShellExecutable(); + yarnExec.setCmd("yarn logs -applicationId " + applicationId + " > " + destFile.getAbsolutePath()); + yarnExec.setName(yarnExec.getCmd()); + + logger.info(yarnExec.getCmd()); + kylinConfig.getCliCommandExecutor().execute(yarnExec.getCmd(), null); + } + } + + private void addRequired(String record) { + logger.info("adding required resource {}", record); + requiredResources.add(record); + } + + public static void main(String args[]) { + JobInfoExtractor extractor = new JobInfoExtractor(); + extractor.execute(args); + } +}
