KYLIN-1614 Diagnosis tool supports 1.3
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b2b9147 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b2b9147 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b2b9147 Branch: refs/heads/1.5.x-HBase1.x Commit: 7b2b9147db397b85ec45d17bb3787c9ff0f27a68 Parents: 51c275f Author: lidongsjtu <lid...@apache.org> Authored: Thu May 5 18:39:31 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Thu May 5 18:39:51 2016 +0800 ---------------------------------------------------------------------- .../kylin/tool/AbstractInfoExtractor.java | 1 + .../apache/kylin/tool/CubeMetaExtractor.java | 53 ++++++++++------ .../org/apache/kylin/tool/DiagnosisInfoCLI.java | 1 + .../java/org/apache/kylin/tool/ToolUtil.java | 58 ----------------- .../kylin/tool/util/ResourceStoreUtil.java | 67 ++++++++++++++++++++ .../org/apache/kylin/tool/util/ToolUtil.java | 58 +++++++++++++++++ 6 files changed, 162 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java index 0706143..9e4e83c 100644 --- a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java @@ -34,6 +34,7 @@ import org.apache.kylin.common.KylinVersion; import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.common.util.ZipFileUtils; +import org.apache.kylin.tool.util.ToolUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java index bfe56e2..a8b732b 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java @@ -19,7 +19,6 @@ package org.apache.kylin.tool; import java.io.File; -import java.io.IOException; import java.util.List; import org.apache.commons.cli.Option; @@ -28,7 +27,6 @@ import org.apache.commons.cli.OptionGroup; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.ResourceTool; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeInstance; @@ -38,6 +36,7 @@ import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.job.dao.ExecutableDao; import org.apache.kylin.job.dao.ExecutablePO; import org.apache.kylin.job.exception.PersistentException; +import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.badquery.BadQueryHistoryManager; import org.apache.kylin.metadata.model.DataModelDesc; @@ -51,6 +50,7 @@ import org.apache.kylin.metadata.realization.RealizationRegistry; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.storage.hybrid.HybridInstance; import org.apache.kylin.storage.hybrid.HybridManager; +import org.apache.kylin.tool.util.ResourceStoreUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -178,30 +178,27 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { } try { - ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); - ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest)); + KylinConfig srcConfig = KylinConfig.getInstanceFromEnv(); + KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dest); - for (String path : requiredResources) { - ResourceTool.copyR(src, dst, path); - } + ResourceStoreUtil.copy(srcConfig, dstConfig, requiredResources); - for (String path : optionalResources) { - try { - ResourceTool.copyR(src, dst, path); - } catch (Exception e) { - logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it."); - } + try { + ResourceStoreUtil.copy(srcConfig, dstConfig, optionalResources); + } catch (Exception e) { + logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it."); } + ResourceStore dstStore = ResourceStore.getStore(dstConfig); for (CubeInstance cube : cubesToTrimAndSave) { - CubeInstance trimmedCube = CubeInstance.getCopyOf(cube); + CubeInstance trimmedCube = copyCubeInstance(cube); trimmedCube.getSegments().clear(); trimmedCube.setUuid(cube.getUuid()); - dst.putResource(trimmedCube.getResourcePath(), trimmedCube, CubeManager.CUBE_SERIALIZER); + dstStore.putResource(trimmedCube.getResourcePath(), trimmedCube, CubeManager.CUBE_SERIALIZER); } - } catch (IOException e) { - throw new RuntimeException("IOException", e); + } catch (Exception e) { + throw new RuntimeException("Exception", e); } } @@ -209,6 +206,20 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization()); } + // do not call CubeInstance.getCopyOf() to keep backward compatible with 1.3.x + private static CubeInstance copyCubeInstance(CubeInstance cubeInstance) { + CubeInstance newCube = new CubeInstance(); + newCube.setName(cubeInstance.getName()); + newCube.setSegments(cubeInstance.getSegments()); + newCube.setDescName(cubeInstance.getDescName()); + newCube.setStatus(cubeInstance.getStatus()); + newCube.setOwner(cubeInstance.getOwner()); + newCube.setCost(cubeInstance.getCost()); + newCube.setCreateTimeUTC(System.currentTimeMillis()); + newCube.updateRandomUuid(); + return newCube; + } + // private void dealWithStreaming(CubeInstance cube) { // streamingManager = StreamingManager.getInstance(kylinConfig); // for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) { @@ -219,6 +230,10 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { // } // } + private static String concatCubeDescResourcePath(String descName) { + return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX; + } + private void retrieveResourcePath(IRealization realization) { logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType()); @@ -238,7 +253,9 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { } addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); - addRequired(CubeDesc.concatResourcePath(cubeDesc.getName())); + + // backward compatible with 1.3 + addRequired(concatCubeDescResourcePath(cubeDesc.getName())); if (includeSegments) { addRequired(CubeInstance.concatResourcePath(cube.getName())); http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java index 884a9d4..be41e74 100644 --- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java @@ -25,6 +25,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.io.FileUtils; import org.apache.kylin.common.util.OptionsHelper; +import org.apache.kylin.tool.util.ToolUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java b/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java deleted file mode 100644 index 494f4fe..0000000 --- a/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.kylin.tool; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.engine.mr.HadoopUtil; -import org.apache.kylin.storage.hbase.HBaseConnection; - -public class ToolUtil { - public static String getConfFolder() { - final String CONF = "conf"; - String path = System.getProperty(KylinConfig.KYLIN_CONF); - if (StringUtils.isNotEmpty(path)) { - return path; - } - path = KylinConfig.getKylinHome(); - if (StringUtils.isNotEmpty(path)) { - return path + File.separator + CONF; - } - return null; - } - - public static String getHBaseMetaStoreId() throws IOException { - try (final HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()))) { - final String metaStoreName = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); - final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName)); - return desc.getValue(HBaseConnection.HTABLE_UUID_TAG); - } catch (Exception e) { - return null; - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java b/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java new file mode 100644 index 0000000..c7fd4d9 --- /dev/null +++ b/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.kylin.tool.util; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.List; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.RawResource; +import org.apache.kylin.common.persistence.ResourceStore; + +/** + * Created by dongli on 5/5/16. + */ +public class ResourceStoreUtil { + public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws Exception { + ResourceStore src = ResourceStore.getStore(srcConfig); + ResourceStore dst = ResourceStore.getStore(dstConfig); + for (String path : paths) { + rCopy(src, dst, path); + } + } + + public static void rCopy(ResourceStore src, ResourceStore dst, String path) throws Exception { + Method listResourceMethod = ResourceStore.class.getMethod("listResources", String.class); + Iterable<String> children = (Iterable<String>) listResourceMethod.invoke(src, path); + + // case of resource (not a folder) + if (children == null) { + try { + RawResource res = src.getResource(path); + if (res != null) { + dst.putResource(path, res.inputStream, res.timestamp); + res.inputStream.close(); + } else { + System.out.println("Resource not exist for " + path); + } + } catch (Exception ex) { + System.err.println("Failed to open " + path); + ex.printStackTrace(); + } + } + // case of folder + else { + for (String child : children) + rCopy(src, dst, child); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java new file mode 100644 index 0000000..3fb2c6c --- /dev/null +++ b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.kylin.tool.util; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.engine.mr.HadoopUtil; +import org.apache.kylin.storage.hbase.HBaseConnection; + +public class ToolUtil { + public static String getConfFolder() { + final String CONF = "conf"; + String path = System.getProperty(KylinConfig.KYLIN_CONF); + if (StringUtils.isNotEmpty(path)) { + return path; + } + path = KylinConfig.getKylinHome(); + if (StringUtils.isNotEmpty(path)) { + return path + File.separator + CONF; + } + return null; + } + + public static String getHBaseMetaStoreId() throws IOException { + try (final HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()))) { + final String metaStoreName = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); + final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName)); + return desc.getValue(HBaseConnection.HTABLE_UUID_TAG); + } catch (Exception e) { + return null; + } + } + +}
