Repository: kylin Updated Branches: refs/heads/master bc46e29a6 -> 9ca2b8e44
Add HBase usage to Diagnosis tool Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ca2b8e4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ca2b8e4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ca2b8e4 Branch: refs/heads/master Commit: 9ca2b8e445ab5b78198fb3bfca5f5eed3f2a4123 Parents: bc46e29 Author: lidongsjtu <lid...@apache.org> Authored: Thu Apr 21 15:58:10 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Thu Apr 21 15:58:20 2016 +0800 ---------------------------------------------------------------------- tool/pom.xml | 12 ++ .../apache/kylin/tool/CubeMetaExtractor.java | 5 +- .../org/apache/kylin/tool/DiagnosisInfoCLI.java | 16 +- .../apache/kylin/tool/HBaseUsageExtractor.java | 207 +++++++++++++++++++ 4 files changed, 237 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9ca2b8e4/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml index 635c606..7df3f0c 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -59,6 +59,18 @@ <artifactId>kylin-invertedindex</artifactId> <version>${project.parent.version}</version> </dependency> + + <!--Env--> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <scope>provided</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/kylin/blob/9ca2b8e4/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java index 649eb71..af7504b 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java @@ -276,6 +276,7 @@ public class CubeMetaExtractor extends AbstractApplication { if (includeSegments) { addRequired(CubeInstance.concatResourcePath(cube.getName())); for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) { + addRequired(segment.getStatisticsResourcePath()); if (includeSegmentDetails) { for (String dictPat : segment.getDictionaryPaths()) { addRequired(dictPat); @@ -322,9 +323,9 @@ public class CubeMetaExtractor extends AbstractApplication { retrieveResourcePath(iRealization); } } else if (realization instanceof IIInstance) { - throw new IllegalStateException("Does not support extract II instance or hybrid that contains II"); + logger.warn("Does not support extract II instance or hybrid that contains II"); } else { - throw new IllegalStateException("Unknown realization type: " + realization.getType()); + logger.warn("Unknown realization type: " + realization.getType()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/9ca2b8e4/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java index a155b8d..de9b6f0 100644 --- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java @@ -58,12 +58,17 @@ public class DiagnosisInfoCLI extends AbstractApplication { @SuppressWarnings("static-access") private static final Option OPTION_INCLUDE_CONF = OptionBuilder.withArgName("includeConf").hasArg().isRequired(false).withDescription("Specify whether to include conf files to extract. Default true.").create("includeConf"); + @SuppressWarnings("static-access") + private static final Option OPTION_INCLUDE_HBASE = OptionBuilder.withArgName("includeHBase").hasArg().isRequired(false).withDescription("Specify whether to include hbase files to extract. Default true.").create("includeHBase"); + private CubeMetaExtractor cubeMetaExtractor; + private HBaseUsageExtractor hBaseUsageExtractor; private Options options; private String exportDest; public DiagnosisInfoCLI() { cubeMetaExtractor = new CubeMetaExtractor(); + hBaseUsageExtractor = new HBaseUsageExtractor(); options = new Options(); options.addOption(OPTION_LOG_PERIOD); @@ -96,7 +101,9 @@ public class DiagnosisInfoCLI extends AbstractApplication { } // create new folder to contain the output - exportDest = exportDest + "diagnosis_" + new SimpleDateFormat("YYYY_MM_dd_HH_mm_ss").format(new Date()) + "/"; + if (new File(exportDest).exists()) { + exportDest = exportDest + "diagnosis_" + new SimpleDateFormat("YYYY_MM_dd_HH_mm_ss").format(new Date()) + "/"; + } // export cube metadata String[] cubeMetaArgs = { "-destDir", exportDest + "metadata", "-project", project }; @@ -105,6 +112,13 @@ public class DiagnosisInfoCLI extends AbstractApplication { int logPeriod = optionsHelper.hasOption(OPTION_LOG_PERIOD) ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_LOG_PERIOD)) : DEFAULT_LOG_PERIOD; boolean compress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : true; boolean includeConf = optionsHelper.hasOption(OPTION_INCLUDE_CONF) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CONF)) : true; + boolean includeHBase = optionsHelper.hasOption(OPTION_INCLUDE_HBASE) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_HBASE)) : true; + + // export HBase + if (includeHBase) { + String[] hbaseArgs = { "-destDir", exportDest + "hbase", "-project", project }; + hBaseUsageExtractor.execute(hbaseArgs); + } // export logs if (logPeriod > 0) { http://git-wip-us.apache.org/repos/asf/kylin/blob/9ca2b8e4/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java new file mode 100644 index 0000000..813a885 --- /dev/null +++ b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java @@ -0,0 +1,207 @@ +package org.apache.kylin.tool; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.List; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.CliCommandExecutor; +import org.apache.kylin.common.util.OptionsHelper; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.project.RealizationEntry; +import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.metadata.realization.RealizationRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class HBaseUsageExtractor extends AbstractApplication { + + private static final Logger logger = LoggerFactory.getLogger(CubeMetaExtractor.class); + @SuppressWarnings("static-access") + private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube"); + @SuppressWarnings("static-access") + private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project"); + @SuppressWarnings("static-access") + private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(false).withDescription("specify the dest dir to save the related metadata").create("destDir"); + + private List<String> htables = Lists.newArrayList(); + private Configuration conf; + private CubeManager cubeManager; + private RealizationRegistry realizationRegistry; + private KylinConfig kylinConfig; + private ProjectManager projectManager; + private Options options = null; + + public HBaseUsageExtractor() { + options = new Options(); + + OptionGroup realizationOrProject = new OptionGroup(); + realizationOrProject.addOption(OPTION_CUBE); + realizationOrProject.addOption(OPTION_PROJECT); + realizationOrProject.setRequired(true); + + options.addOptionGroup(realizationOrProject); + options.addOption(OPTION_DEST); + + conf = HBaseConfiguration.create(); + } + + public static void main(String[] args) { + HBaseUsageExtractor extractor = new HBaseUsageExtractor(); + extractor.execute(args); + } + + @Override + protected Options getOptions() { + return options; + } + + private String getHBaseMasterUrl() { + String host = conf.get("hbase.master.info.bindAddress"); + String port = conf.get("hbase.master.info.port"); + return "http://"; + host + ":" + port + "/"; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + String dest = null; + if (optionsHelper.hasOption(OPTION_DEST)) { + dest = optionsHelper.getOptionValue(OPTION_DEST); + } + + if (org.apache.commons.lang3.StringUtils.isEmpty(dest)) { + throw new RuntimeException("destDir is not set, exit directly without extracting"); + } + + if (!dest.endsWith("/")) { + dest = dest + "/"; + } + + kylinConfig = KylinConfig.getInstanceFromEnv(); + cubeManager = CubeManager.getInstance(kylinConfig); + realizationRegistry = RealizationRegistry.getInstance(kylinConfig); + projectManager = ProjectManager.getInstance(kylinConfig); + + if (optionsHelper.hasOption(OPTION_PROJECT)) { + String projectName = optionsHelper.getOptionValue(OPTION_PROJECT); + ProjectInstance projectInstance = projectManager.getProject(projectName); + if (projectInstance == null) { + throw new IllegalArgumentException("Project " + projectName + " does not exist"); + } + List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries(); + for (RealizationEntry realizationEntry : realizationEntries) { + retrieveResourcePath(getRealization(realizationEntry)); + } + } else if (optionsHelper.hasOption(OPTION_CUBE)) { + String cubeName = optionsHelper.getOptionValue(OPTION_CUBE); + IRealization realization; + if ((realization = cubeManager.getRealization(cubeName)) != null) { + retrieveResourcePath(realization); + } else { + throw new IllegalArgumentException("No cube found with name of " + cubeName); + } + } + + extractCommonInfo(dest); + extractHTables(dest); + + logger.info("Extracted metadata files located at: " + new File(dest).getAbsolutePath()); + } + + private void extractHTables(String dest) throws IOException { + logger.info("These htables are going to be extracted:"); + for (String htable : htables) { + logger.info(htable + "(required)"); + } + + File tableDir = new File(dest, "table"); + FileUtils.forceMkdir(tableDir); + + for (String htable : htables) { + try { + URL srcUrl = new URL(getHBaseMasterUrl() + "table.jsp?name=" + htable); + File destFile = new File(tableDir, htable + ".html"); + FileUtils.copyURLToFile(srcUrl, destFile); + } catch (Exception e) { + logger.warn("HTable " + htable + "info fetch failed: ", e); + } + } + } + + private void extractCommonInfo(String dest) throws IOException { + logger.info("The hbase master info/conf are going to be extracted..."); + + // hbase master page + try { + File masterDir = new File(dest, "master"); + FileUtils.forceMkdir(masterDir); + URL srcMasterUrl = new URL(getHBaseMasterUrl() + "master-status"); + File masterDestFile = new File(masterDir, "master-status.html"); + FileUtils.copyURLToFile(srcMasterUrl, masterDestFile); + } catch (Exception e) { + logger.warn("HBase Master status fetch failed: ", e); + } + + // hbase conf + try { + File confDir = new File(dest, "conf"); + FileUtils.forceMkdir(confDir); + URL srcConfUrl = new URL(getHBaseMasterUrl() + "conf"); + File destConfFile = new File(confDir, "hbase-conf.xml"); + FileUtils.copyURLToFile(srcConfUrl, destConfFile); + } catch (Exception e) { + logger.warn("HBase conf fetch failed: ", e); + } + + // hbase hdfs status + try { + File hdfsDir = new File(dest, "hdfs"); + FileUtils.forceMkdir(hdfsDir); + CliCommandExecutor cliCommandExecutor = kylinConfig.getCliCommandExecutor(); + String output = cliCommandExecutor.execute("hadoop fs -ls -R " + conf.get("hbase.rootdir") + "/data/default/KYLIN_*").getSecond(); + FileUtils.writeStringToFile(new File(hdfsDir, "hdfs-files.list"), output); + output = cliCommandExecutor.execute("hadoop fs -ls -R " + conf.get("hbase.rootdir") + "/data/default/kylin_*").getSecond(); + FileUtils.writeStringToFile(new File(hdfsDir, "hdfs-files.list"), output, true); + } catch (Exception e) { + logger.warn("HBase hdfs status fetch failed: ", e); + } + } + + private IRealization getRealization(RealizationEntry realizationEntry) { + return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization()); + } + + private void retrieveResourcePath(IRealization realization) { + + logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType()); + + if (realization instanceof CubeInstance) { + CubeInstance cube = (CubeInstance) realization; + for (CubeSegment segment : cube.getSegments()) { + addHTable(segment.getStorageLocationIdentifier()); + } + } else { + logger.warn("Unknown realization type: " + realization.getType()); + } + } + + private void addHTable(String record) { + logger.info("adding required resource {}", record); + htables.add(record); + } +}
