KYLIN-1614 Diagnosis CLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eac98036 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eac98036 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eac98036 Branch: refs/heads/1.5.x-HBase1.x Commit: eac98036ddf228daaa71bd41f00179fa493f2759 Parents: f17ba5c Author: lidongsjtu <lid...@apache.org> Authored: Fri May 6 12:23:59 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Fri May 6 17:09:31 2016 +0800 ---------------------------------------------------------------------- build/deploy/diag.sh | 68 ++++++++++++++ build/script/package_diag.sh | 97 ++++++++++++++++++++ tool/pom.xml | 2 +- .../apache/kylin/tool/KylinLogExtractor.java | 13 ++- 4 files changed, 175 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/eac98036/build/deploy/diag.sh ---------------------------------------------------------------------- diff --git a/build/deploy/diag.sh b/build/deploy/diag.sh new file mode 100644 index 0000000..1deeded --- /dev/null +++ b/build/deploy/diag.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dir=$(dirname ${0}) + +# We should set KYLIN_HOME here for multiple tomcat instances that are on the same node. +# In addition, we should set a KYLIN_HOME for the global use as normal. +export KYLIN_HOME=${dir}/../ +source ${dir}/check-env.sh + +if [ $# -gt 1 ] +then + patient="$1" + if [ -z "$patient" ] + then + echo "You need to specify a Project or Job Id for diagnosis." + exit 1 + fi + destDir="$2" + if [ -z "$destDir" ] + then + destDir="$KYLIN_HOME/diagnosis_dump/" + mkdir -p $destDir + fi + + #retrive $hive_dependency and $hbase_dependency + source ${dir}/find-hive-dependency.sh + source ${dir}/find-hbase-dependency.sh + + #retrive $KYLIN_EXTRA_START_OPTS + if [ -f "${dir}/setenv-tool.sh" ] + then source ${dir}/setenv-tool.sh + fi + export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH} + + if [ -f "${KYLIN_HOME}/lib/kylin-diagnosis-*.jar" ]; then + export HBASE_CLASSPATH=$HBASE_CLASSPATH:${KYLIN_HOME}/lib/kylin-diagnosis-*.jar + else + export HBASE_CLASSPATH=$HBASE_CLASSPATH:${KYLIN_HOME}/lib/* + fi + + if [ ${#patient} -gt 31 ]; then + exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir + else + exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project $patient -destDir $destDir + fi + + exit 0 +else + echo "usage: diag.sh Project|JobId [target_path]" + exit 1 +fi http://git-wip-us.apache.org/repos/asf/kylin/blob/eac98036/build/script/package_diag.sh ---------------------------------------------------------------------- diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh new file mode 100755 index 0000000..413bca8 --- /dev/null +++ b/build/script/package_diag.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo "Checking maven..." + +if [ -z "$(command -v mvn)" ] +then + echo "Please install maven first so that Kylin packaging can proceed" + exit 1 +else + echo "maven check passed" +fi + +echo "Checking git..." + +if [ -z "$(command -v git)" ] +then + echo "Please install git first so that Kylin packaging can proceed" + exit 1 +else + echo "git check passed" +fi + +dir=$(dirname ${0}) +cd ${dir}/../.. +version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['` + +if [ "$version" == "" ];then + echo "Failed to identify kylin version (current: `pwd`)" + exit 1 +fi + +echo "kylin version: ${version}" +export version + +#commit id +cat << EOF > build/commit_SHA1 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +EOF +git rev-parse HEAD >> build/commit_SHA1 + +echo "package libraries" +mvn clean install -DskipTests || { exit 1; } + +echo "copy libraries" +rm -rf build/lib +mkdir build/lib +cp tool/target/kylin-tool-${version}-assembly.jar build/lib/kylin-diagnosis-${version}.jar +# Copied file becomes 000 for some env (e.g. my Cygwin) +chmod 644 build/lib/kylin-diagnosis-${version}.jar + +echo 'package tar.gz' +package_name=apache-kylin-${version}-diag +cd build/ +rm -rf ${package_name} +mkdir ${package_name} +cp -r lib ${package_name} +cp deploy/diag.sh ${package_name} +rm -rf lib +find ${package_name} -type d -exec chmod 755 {} \; +find ${package_name} -type f -exec chmod 644 {} \; +find ${package_name} -type f -name "*.sh" -exec chmod 755 {} \; +mkdir -p ../dist +tar -cvzf ../dist/${package_name}.tar.gz ${package_name} +rm -rf ${package_name} + +echo "Library package ready: dist/${package_name}.tar.gz" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/eac98036/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml index 7fc08f4..3f077de 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -74,7 +74,7 @@ <configuration> <minimizeJar>false</minimizeJar> <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>tool</shadedClassifierName> + <shadedClassifierName>assembly</shadedClassifierName> <artifactSet> <excludes> <exclude>io.netty:*</exclude> http://git-wip-us.apache.org/repos/asf/kylin/blob/eac98036/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java index 5351c92..d71e6c6 100644 --- a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java @@ -59,16 +59,21 @@ public class KylinLogExtractor extends AbstractInfoExtractor { logger.info("Start to extract kylin logs in {} days", logPeriod); final File kylinLogDir = new File(KylinConfig.getKylinHome(), "logs"); - final ArrayList<File> logFiles = Lists.newArrayList(); + final ArrayList<File> requiredLogFiles = Lists.newArrayList(); final long logThresholdTime = System.currentTimeMillis() - logPeriod * 24 * 3600 * 1000; - for (File logFile : kylinLogDir.listFiles()) { + final File[] allLogFiles = kylinLogDir.listFiles(); + if (allLogFiles == null || allLogFiles.length == 0) { + return; + } + + for (File logFile : allLogFiles) { if (logFile.lastModified() > logThresholdTime) { - logFiles.add(logFile); + requiredLogFiles.add(logFile); } } - for (File logFile : logFiles) { + for (File logFile : requiredLogFiles) { logger.info("Log file:" + logFile.getAbsolutePath()); if (logFile.exists()) { FileUtils.copyFileToDirectory(logFile, exportDir);
