Repository: kylin Updated Branches: refs/heads/master 4ffe818e2 -> 25081d928
KYLIN-1818: change kafka dep to provided Signed-off-by: shaofengshi <shaofeng...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25081d92 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25081d92 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25081d92 Branch: refs/heads/master Commit: 25081d928ef081c343044af3587dc7e2830a4a46 Parents: 4ffe818 Author: Yiming Liu <liuyiming....@gmail.com> Authored: Fri Jun 24 15:13:14 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Jun 24 15:17:33 2016 +0800 ---------------------------------------------------------------------- assembly/pom.xml | 1 - build/bin/find-kafka-dependency.sh | 51 +++++++++++++++++++++++++++++++++ build/bin/kylin.sh | 4 +++ 3 files changed, 55 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/25081d92/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml index d643b62..d7764a0 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -177,7 +177,6 @@ <artifactId>kafka_2.10</artifactId> <version>${kafka.version}</version> <scope>provided</scope> - <!-- FIXME: Should be provided just like hive and hbase, inflates job jar from 9 MB to 21 MB --> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/kylin/blob/25081d92/build/bin/find-kafka-dependency.sh ---------------------------------------------------------------------- diff --git a/build/bin/find-kafka-dependency.sh b/build/bin/find-kafka-dependency.sh new file mode 100644 index 0000000..c6b9c24 --- /dev/null +++ b/build/bin/find-kafka-dependency.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kafka_home= + +if [ -n "$KAFKA_HOME" ] +then + echo "KAFKA_HOME is set to: $KAFKA_HOME, use it to locate kafka dependencies." + kafka_home=$KAFKA_HOME +fi + +if [ -z "$KAFKA_HOME" ] +then + echo "Couldn't find kafka home. Please set KAFKA_HOME to the path which contains kafka dependencies." + exit 1 +fi + +# works for kafka 9+ +kafka_client=`find -L "$(dirname $kafka_home)" -name 'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` +if [ -z "$kafka_client" ] +then +# works for kafka 8 + kafka_broker=`find -L "$(dirname $kafka_home)" -name 'kafka_[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` + if [ -z "$kafka_broker" ] + then + echo "kafka client lib not found" + exit 1 + else + echo "kafka dependency: $kafka_broker" + export kafka_dependency + fi +else + echo "kafka dependency: $kafka_client" + export kafka_dependency +fi http://git-wip-us.apache.org/repos/asf/kylin/blob/25081d92/build/bin/kylin.sh ---------------------------------------------------------------------- diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh index 2ea3a8c..f3710c3 100644 --- a/build/bin/kylin.sh +++ b/build/bin/kylin.sh @@ -141,6 +141,7 @@ then #retrive $hive_dependency and $hbase_dependency source ${dir}/find-hive-dependency.sh source ${dir}/find-hbase-dependency.sh + source ${dir}/find-kafka-dependency.sh #retrive $KYLIN_EXTRA_START_OPTS if [ -f "${dir}/setenv.sh" ] then source ${dir}/setenv.sh @@ -153,6 +154,7 @@ then hbase ${KYLIN_EXTRA_START_OPTS} \ -Dlog4j.configuration=kylin-log4j.properties\ -Dkylin.hive.dependency=${hive_dependency} \ + -Dkylin.kafka.dependency=${kafka_dependency} \ -Dkylin.hbase.dependency=${hbase_dependency} \ org.apache.kylin.engine.streaming.cli.StreamingCLI $@ > ${KYLIN_HOME}/logs/streaming_$3_$4.log 2>&1 & echo $! > ${KYLIN_HOME}/logs/$3_$4 & echo "streaming started name: $3 id: $4" @@ -187,6 +189,7 @@ then #retrive $hive_dependency and $hbase_dependency source ${dir}/find-hive-dependency.sh source ${dir}/find-hbase-dependency.sh + source ${dir}/find-kafka-dependency.sh #retrive $KYLIN_EXTRA_START_OPTS if [ -f "${dir}/setenv.sh" ] then source ${dir}/setenv.sh @@ -199,6 +202,7 @@ then hbase ${KYLIN_EXTRA_START_OPTS} \ -Dlog4j.configuration=kylin-log4j.properties\ -Dkylin.hive.dependency=${hive_dependency} \ + -Dkyiln.kafka.dependency=${kafka_dependency} \ -Dkylin.hbase.dependency=${hbase_dependency} \ org.apache.kylin.engine.streaming.cli.MonitorCLI $@ > ${KYLIN_HOME}/logs/monitor.log 2>&1 exit 0
