KYLIN-2163 refine kylin scripts
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8e31ec5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8e31ec5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8e31ec5 Branch: refs/heads/yang21-hbase1.x Commit: f8e31ec54dab0a77b3ba2885cf229a99f1c8560c Parents: b4f9179 Author: Yang Li <liy...@apache.org> Authored: Sat Nov 5 12:00:53 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sat Nov 5 12:00:53 2016 +0800 ---------------------------------------------------------------------- build/bin/check-env.sh | 17 ++++------ build/bin/diag.sh | 7 +--- build/bin/find-hbase-dependency.sh | 7 ++-- build/bin/find-hive-dependency.sh | 31 +++++++----------- build/bin/find-kafka-dependency.sh | 2 ++ build/bin/get-properties.sh | 2 ++ build/bin/header.sh | 41 ++++++++++++++++++++++++ build/bin/kylin.sh | 57 +++++++++++++-------------------- build/bin/metastore.sh | 7 +--- build/bin/sample.sh | 3 +- build/bin/setenv.sh | 12 ++++--- 11 files changed, 99 insertions(+), 87 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/check-env.sh ---------------------------------------------------------------------- diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh index 1597c53..9cb7636 100644 --- a/build/bin/check-env.sh +++ b/build/bin/check-env.sh @@ -17,38 +17,33 @@ # limitations under the License. # -#by source +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh if [ -z "$KYLIN_HOME" ] then - echo 'please make sure KYLIN_HOME has been set' - exit 1 + quit 'Please make sure KYLIN_HOME has been set' else echo "KYLIN_HOME is set to ${KYLIN_HOME}" fi if [ -z "$(command -v hbase version)" ] then - echo "Please make sure the user has the privilege to run hbase shell" - exit 1 + quit "Please make sure the user has the privilege to run hbase shell" fi if [ -z "$(command -v hive --version)" ] then - echo "Please make sure the user has the privilege to run hive shell" - exit 1 + quit "Please make sure the user has the privilege to run hive shell" fi if [ -z "$(command -v hadoop version)" ] then - echo "Please make sure the user has the privilege to run hadoop shell" - exit 1 + quit "Please make sure the user has the privilege to run hadoop shell" fi WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.hdfs.working.dir` hadoop fs -mkdir -p $WORKING_DIR if [ $? != 0 ] then - echo "failed to create $WORKING_DIR, Please make sure the user has right to access $WORKING_DIR" - exit 1 + quit "Failed to create $WORKING_DIR. Please make sure the user has right to access $WORKING_DIR" fi http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/diag.sh ---------------------------------------------------------------------- diff --git a/build/bin/diag.sh b/build/bin/diag.sh index 91ba92b..5f3cec3 100644 --- a/build/bin/diag.sh +++ b/build/bin/diag.sh @@ -17,13 +17,8 @@ # limitations under the License. # -# We should set KYLIN_HOME here for multiple tomcat instances that are on the same node. -# In addition, we should set a KYLIN_HOME for the global use as normal. -KYLIN_HOME=`dirname $0`/.. -export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd` -dir="$KYLIN_HOME/bin" +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh -source ${dir}/check-env.sh mkdir -p ${KYLIN_HOME}/logs tomcat_root=${dir}/../tomcat http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/find-hbase-dependency.sh ---------------------------------------------------------------------- diff --git a/build/bin/find-hbase-dependency.sh b/build/bin/find-hbase-dependency.sh index ea9a2e2..7dbb53b 100644 --- a/build/bin/find-hbase-dependency.sh +++ b/build/bin/find-hbase-dependency.sh @@ -17,6 +17,8 @@ # limitations under the License. # +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh + hbase_classpath=`hbase classpath` # special handling for Amazon EMR, to prevent re-init of hbase-setenv @@ -38,10 +40,9 @@ done if [ -z "$hbase_common_path" ] then - echo "hbase-common lib not found" - exit 1 + quit "hbase-common lib not found" fi hbase_dependency=${hbase_common_path} -echo "hbase dependency: $hbase_dependency" +verbose "hbase dependency: $hbase_dependency" export hbase_dependency http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/find-hive-dependency.sh ---------------------------------------------------------------------- diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh index d17b47e..69b63a4 100644 --- a/build/bin/find-hive-dependency.sh +++ b/build/bin/find-hive-dependency.sh @@ -17,24 +17,18 @@ # limitations under the License. # -if [ -z "$KYLIN_HOME" ] -then - echo 'please make sure KYLIN_HOME has been set' - exit 1 -else - echo "KYLIN_HOME is set to ${KYLIN_HOME}" -fi - +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.client` hive_env= +echo Retrieving hive dependency... if [ "${client_mode}" == "beeline" ] then beeline_params=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.beeline.params` - hive_env=`beeline ${beeline_params} --outputformat=dsv -e set | grep 'env:CLASSPATH'` + hive_env=`beeline ${beeline_params} --outputformat=dsv -e set 2>&1 | grep 'env:CLASSPATH' ` else - hive_env=`hive -e set | grep 'env:CLASSPATH'` + hive_env=`hive -e set 2>&1 | grep 'env:CLASSPATH'` fi hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk -F '=' '{print $2}'` @@ -44,7 +38,7 @@ hive_exec_path= if [ -n "$HIVE_CONF" ] then - echo "HIVE_CONF is set to: $HIVE_CONF, use it to locate hive configurations." + verbose "HIVE_CONF is set to: $HIVE_CONF, use it to locate hive configurations." hive_conf_path=$HIVE_CONF fi @@ -69,14 +63,13 @@ done if [ -z "$hive_conf_path" ] then - echo "Couldn't find hive configuration directory. Please set HIVE_CONF to the path which contains hive-site.xml." - exit 1 + quit "Couldn't find hive configuration directory. Please set HIVE_CONF to the path which contains hive-site.xml." fi # in some versions of hive hcatalog is not in hive's classpath, find it separately if [ -z "$HCAT_HOME" ] then - echo "HCAT_HOME not found, try to find hcatalog path from hadoop home" + verbose "HCAT_HOME not found, try to find hcatalog path from hadoop home" hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'` hive_home=`echo $hive_exec_path | awk -F '/lib/' '{print $1}'` is_aws=`uname -r | grep amzn` @@ -90,11 +83,10 @@ then # special handling for Amazon EMR hcatalog_home=/usr/lib/hive-hcatalog else - echo "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path." - exit 1 + quit "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path." fi else - echo "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:" + verbose "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:" hcatalog_home=${HCAT_HOME} fi @@ -102,12 +94,11 @@ hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*.jar" 2>&1 if [ -z "$hcatalog" ] then - echo "hcatalog lib not found" - exit 1 + quit "hcatalog lib not found" fi hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'` hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog} -echo "hive dependency: $hive_dependency" +verbose "hive dependency: $hive_dependency" export hive_dependency http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/find-kafka-dependency.sh ---------------------------------------------------------------------- diff --git a/build/bin/find-kafka-dependency.sh b/build/bin/find-kafka-dependency.sh index c6b9c24..40c8239 100644 --- a/build/bin/find-kafka-dependency.sh +++ b/build/bin/find-kafka-dependency.sh @@ -17,6 +17,8 @@ # limitations under the License. # +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh + kafka_home= if [ -n "$KAFKA_HOME" ] http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/get-properties.sh ---------------------------------------------------------------------- diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh index b0c7387..45bc49e 100755 --- a/build/bin/get-properties.sh +++ b/build/bin/get-properties.sh @@ -17,6 +17,8 @@ # limitations under the License. # +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh + if [ $# != 1 ] then echo 'invalid input' http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/header.sh ---------------------------------------------------------------------- diff --git a/build/bin/header.sh b/build/bin/header.sh new file mode 100644 index 0000000..949ea95 --- /dev/null +++ b/build/bin/header.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# source me + +if [[ "$dir" == "" ]] +then + dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) + + # set KYLIN_HOME with consideration for multiple instances that are on the same node + KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"} + export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd` + dir="$KYLIN_HOME/bin" + + function quit { + echo "$@" + exit 1 + } + + function verbose { + if [[ -n "$verbose" ]]; then + echo "$@" + fi + } +fi http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/kylin.sh ---------------------------------------------------------------------- diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh index 244838b..f4a99da 100644 --- a/build/bin/kylin.sh +++ b/build/bin/kylin.sh @@ -17,12 +17,10 @@ # limitations under the License. # -dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) +# set verbose=true to print more logs during start up +verbose=${verbose:-""} -# set KYLIN_HOME with consideration for multiple instances that are on the same node -KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"} -export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd` -dir="$KYLIN_HOME/bin" +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh source ${dir}/check-env.sh mkdir -p ${KYLIN_HOME}/logs @@ -34,8 +32,8 @@ function retrieveDependency() { source ${dir}/find-hbase-dependency.sh #retrive $KYLIN_EXTRA_START_OPTS - if [ -f "${dir}/setenv.sh" ] - then source ${dir}/setenv.sh + if [ -f "${dir}/setenv.sh" ]; then + source ${dir}/setenv.sh fi export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX} @@ -71,10 +69,9 @@ then spring_profile=`sh ${dir}/get-properties.sh kylin.security.profile` if [ -z "$spring_profile" ] then - echo 'please set kylin.security.profile in kylin.properties, options are: testing, ldap, saml.' - exit 1 + quit 'please set kylin.security.profile in kylin.properties, options are: testing, ldap, saml.' else - echo "kylin.security.profile is set to $spring_profile" + verbose "kylin.security.profile is set to $spring_profile" fi retrieveDependency @@ -85,15 +82,13 @@ then if [ -z "$KYLIN_REST_ADDRESS" ] then kylin_rest_address=`hostname -f`":"`grep "<Connector port=" ${tomcat_root}/conf/server.xml |grep protocol=\"HTTP/1.1\" | cut -d '=' -f 2 | cut -d \" -f 2` - echo "KYLIN_REST_ADDRESS not found, will use ${kylin_rest_address}" else - echo "KYLIN_REST_ADDRESS is set to: $KYLIN_REST_ADDRESS" kylin_rest_address=$KYLIN_REST_ADDRESS fi + verbose "kylin.rest.address is set to ${kylin_rest_address}" #debug if encounter NoClassDefError - echo "hbase classpath is:" - hbase classpath + verbose "kylin classpath is: $(hbase classpath)" # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh hbase ${KYLIN_EXTRA_START_OPTS} \ @@ -110,9 +105,11 @@ then -Dkylin.rest.address=${kylin_rest_address} \ -Dspring.profiles.active=${spring_profile} \ org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.out 2>&1 & echo $! > ${KYLIN_HOME}/pid & - echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\"" - echo "Please visit http://<ip>:7070/kylin" - echo "You can check the log at ${KYLIN_HOME}/logs/kylin.log" + + echo "" + echo "A new Kylin instance is started by $USER. To stop it, run 'kylin.sh stop'" + echo "Check the log at ${KYLIN_HOME}/logs/kylin.log" + echo "Web UI is at http://<hostname>:7070/kylin" exit 0 # stop command @@ -123,18 +120,16 @@ then PID=`cat $KYLIN_HOME/pid` if ps -p $PID > /dev/null then - echo "stopping Kylin:$PID" + echo "Stopping Kylin: $PID" kill $PID rm ${KYLIN_HOME}/pid exit 0 else - echo "Kylin is not running, please check" - exit 1 + quit "Kylin is not running" fi else - echo "Kylin is not running, please check" - exit 1 + quit "Kylin is not running" fi # streaming command @@ -142,8 +137,7 @@ elif [ "$1" == "streaming" ] then if [ $# -lt 4 ] then - echo "invalid input args $@" - exit -1 + quit "Invalid input args $@" fi if [ "$2" == "start" ] then @@ -162,17 +156,15 @@ then elif [ "$2" == "stop" ] then if [ ! -f "${KYLIN_HOME}/$3_$4" ] - then - echo "streaming is not running, please check" - exit 1 + then + quit "Streaming is not running" fi pid=`cat ${KYLIN_HOME}/$3_$4` if [ "$pid" = "" ] then - echo "streaming is not running, please check" - exit 1 + quit "Streaming is not running" else - echo "stopping streaming:$pid" + echo "Stopping streaming: $pid" kill $pid fi rm ${KYLIN_HOME}/$3_$4 @@ -184,8 +176,6 @@ then # monitor command elif [ "$1" == "monitor" ] then - echo "monitor job" - retrieveDependency source ${dir}/find-kafka-dependency.sh @@ -222,6 +212,5 @@ then exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=kylin-log4j.properties "$@" else - echo "usage: kylin.sh start or kylin.sh stop" - exit 1 + quit "Usage: 'kylin.sh start' or 'kylin.sh stop'" fi http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/metastore.sh ---------------------------------------------------------------------- diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh index a7a9e27..72afbae 100755 --- a/build/bin/metastore.sh +++ b/build/bin/metastore.sh @@ -24,12 +24,7 @@ # take a look at SandboxMetastoreCLI -dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) - -# set KYLIN_HOME with consideration for multiple instances that are on the same node -KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"} -export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd` -dir="$KYLIN_HOME/bin" +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh source ${dir}/check-env.sh http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/sample.sh ---------------------------------------------------------------------- diff --git a/build/bin/sample.sh b/build/bin/sample.sh index fdb8601..e420df7 100644 --- a/build/bin/sample.sh +++ b/build/bin/sample.sh @@ -17,8 +17,7 @@ # limitations under the License. # -dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) -export KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"} +source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh source ${dir}/check-env.sh job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar` http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/setenv.sh ---------------------------------------------------------------------- diff --git a/build/bin/setenv.sh b/build/bin/setenv.sh index 24084f3..f6cf210 100755 --- a/build/bin/setenv.sh +++ b/build/bin/setenv.sh @@ -17,6 +17,8 @@ # limitations under the License. # +# source me + # (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings) # uncomment following to for it to take effect export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss1024K -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M" @@ -35,21 +37,21 @@ then echo "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}" KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}" else - echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings: ${KYLIN_JVM_SETTINGS}" + verbose "KYLIN_JVM_SETTINGS is not set, using default jvm settings: ${KYLIN_JVM_SETTINGS}" fi if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ] then - echo "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}" + verbose "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}" KYLIN_EXTRA_START_OPTS="${KYLIN_DEBUG_SETTINGS} ${KYLIN_EXTRA_START_OPTS}" else - echo "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging" + verbose "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging" fi if [ ! -z "${KYLIN_LD_LIBRARY_SETTINGS}" ] then - echo "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}" + verbose "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}" KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}" else - echo "KYLIN_LD_LIBRARY_SETTINGS is not set, Usually it's okay unless you want to specify your own native path" + verbose "KYLIN_LD_LIBRARY_SETTINGS is not set, it is okay unless you want to specify your own native path" fi
