KYLIN-1153 Tool for updating CubeDesc metadata Signed-off-by: honma <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9f60f476 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9f60f476 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9f60f476 Branch: refs/heads/2.x-staging Commit: 9f60f4760cca66d67e507c33a8d98433728a5076 Parents: cfa97f6 Author: lidongsjtu <[email protected]> Authored: Mon Nov 30 10:44:47 2015 +0800 Committer: honma <[email protected]> Committed: Fri Dec 4 16:29:06 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/ClassUtil.java | 1 + .../job/upgrade/CubeDescSignatureUpdate.java | 117 +++++++++++++++++++ .../hbase/util/DeployCoprocessorCLI.java | 7 +- 3 files changed, 121 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9f60f476/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java index 93790e6..ce1f014 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java @@ -50,6 +50,7 @@ public class ClassUtil { classRenameMap.put("org.apache.kylin.job.common.HadoopShellExecutable", "org.apache.kylin.engine.mr.common.HadoopShellExecutable"); classRenameMap.put("org.apache.kylin.job.common.MapReduceExecutable", "org.apache.kylin.engine.mr.common.MapReduceExecutable"); classRenameMap.put("org.apache.kylin.job.cube.CubingJob", "org.apache.kylin.engine.mr.CubingJob"); + classRenameMap.put("org.apache.kylin.job.invertedindex.IIJob", "org.apache.kylin.engine.mr.invertedindex.IIJob"); classRenameMap.put("org.apache.kylin.job.cube.GarbageCollectionStep", "org.apache.kylin.storage.hbase.steps.DeprecatedGCStep"); classRenameMap.put("org.apache.kylin.job.cube.MergeDictionaryStep", "org.apache.kylin.engine.mr.steps.MergeDictionaryStep"); classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterBuildStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep"); http://git-wip-us.apache.org/repos/asf/kylin/blob/9f60f476/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java new file mode 100644 index 0000000..178f1b0 --- /dev/null +++ b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.job.upgrade; + +import com.google.common.collect.Lists; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.model.CubeDesc; + +import java.util.List; + +/** + * Created by dongli on 11/17/15. + */ +public class CubeDescSignatureUpdate { + private KylinConfig config = null; + private ResourceStore store; + private String[] cubeNames; + private List<String> updatedResources = Lists.newArrayList(); + private List<String> errorMsgs = Lists.newArrayList(); + + private static final Log logger = LogFactory.getLog(CubeDescSignatureUpdate.class); + + public CubeDescSignatureUpdate(String[] cubes) { + config = KylinConfig.getInstanceFromEnv(); + store = ResourceStore.getStore(config); + cubeNames = cubes; + } + + public void update() { + logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); + CubeDescManager cubeDescManager = CubeDescManager.getInstance(config); + List<CubeDesc> cubeDescs; + if (ArrayUtils.isEmpty(cubeNames)) { + cubeDescs = cubeDescManager.listAllDesc(); + } else { + String[] names = cubeNames[0].split(","); + if (ArrayUtils.isEmpty(names)) + return; + cubeDescs = Lists.newArrayListWithCapacity(names.length); + for (String name : names) { + cubeDescs.add(cubeDescManager.getCubeDesc(name)); + } + } + for (CubeDesc cubeDesc : cubeDescs) { + updateCubeDesc(cubeDesc); + } + } + + public static void main(String args[]) { + if (args != null && args.length != 0 && args.length != 1) { + System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; e.g, cube1,cube2 "); + return; + } + + CubeDescSignatureUpdate metadataUpgrade = new CubeDescSignatureUpdate(args); + metadataUpgrade.update(); + + logger.info("================================================================="); + logger.info("Run CubeDescSignatureUpdate completed;"); + + if (!metadataUpgrade.updatedResources.isEmpty()) { + logger.info("Following resources are updated successfully:"); + for (String s : metadataUpgrade.updatedResources) { + logger.info(s); + } + } else { + logger.warn("No resource updated."); + } + + if (!metadataUpgrade.errorMsgs.isEmpty()) { + logger.info("Here are the error/warning messages, you may need to check:"); + for (String s : metadataUpgrade.errorMsgs) { + logger.warn(s); + } + } else { + logger.info("No error or warning messages; The update succeeds."); + } + + logger.info("================================================================="); + } + + private void updateCubeDesc(CubeDesc cubeDesc) { + try { + String calculatedSign = cubeDesc.calculateSignature(); + if (!cubeDesc.getSignature().equals(calculatedSign)) + { + cubeDesc.setSignature(calculatedSign); + store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); + updatedResources.add(cubeDesc.getResourcePath()); + } + } catch (Exception e) { + e.printStackTrace(); + errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9f60f476/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java index 15dc993..a5ae09c 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java @@ -282,10 +282,9 @@ public class DeployCoprocessorCLI { } String jarPath = valueMatcher.group(1).trim(); - //String clsName = valueMatcher.group(2).trim(); - //if (CubeObserverClass.equals(clsName)) { - result.add(jarPath); - //} + if (StringUtils.isNotEmpty(jarPath)) { + result.add(jarPath); + } } }
