KYLIN-1818 change kafka dependency to provided Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4e060e7b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4e060e7b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4e060e7b
Branch: refs/heads/KYLIN-1726 Commit: 4e060e7bc89f757a138dfa5beba2a43209e91371 Parents: 42dafc1 Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Sep 2 18:58:11 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Wed Sep 14 16:34:36 2016 +0800 ---------------------------------------------------------------------- build/bin/find-kafka-dependency.sh | 12 ++++++------ build/bin/kylin.sh | 2 ++ .../kylin/engine/mr/common/AbstractHadoopJob.java | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4e060e7b/build/bin/find-kafka-dependency.sh ---------------------------------------------------------------------- diff --git a/build/bin/find-kafka-dependency.sh b/build/bin/find-kafka-dependency.sh index c6b9c24..7349360 100644 --- a/build/bin/find-kafka-dependency.sh +++ b/build/bin/find-kafka-dependency.sh @@ -32,20 +32,20 @@ then fi # works for kafka 9+ -kafka_client=`find -L "$(dirname $kafka_home)" -name 'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` -if [ -z "$kafka_client" ] +kafka_dependency=`find -L $kafka_home -name 'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` +if [ -z "$kafka_dependency" ] then # works for kafka 8 - kafka_broker=`find -L "$(dirname $kafka_home)" -name 'kafka_[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` - if [ -z "$kafka_broker" ] + kafka_dependency=`find -L $kafka_home -name 'kafka_[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` + if [ -z "$kafka_dependency" ] then echo "kafka client lib not found" exit 1 else - echo "kafka dependency: $kafka_broker" + echo "kafka dependency: $kafka_dependency" export kafka_dependency fi else - echo "kafka dependency: $kafka_client" + echo "kafka dependency: $kafka_dependency" export kafka_dependency fi http://git-wip-us.apache.org/repos/asf/kylin/blob/4e060e7b/build/bin/kylin.sh ---------------------------------------------------------------------- diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh index 201b5b6..7a9d2a1 100644 --- a/build/bin/kylin.sh +++ b/build/bin/kylin.sh @@ -31,6 +31,7 @@ function retrieveDependency() { #retrive $hive_dependency and $hbase_dependency source ${dir}/find-hive-dependency.sh source ${dir}/find-hbase-dependency.sh + source ${dir}/find-kafka-dependency.sh #retrive $KYLIN_EXTRA_START_OPTS if [ -f "${dir}/setenv.sh" ] @@ -106,6 +107,7 @@ then -Djava.io.tmpdir=${tomcat_root}/temp \ -Dkylin.hive.dependency=${hive_dependency} \ -Dkylin.hbase.dependency=${hbase_dependency} \ + -Dkylin.kafka.dependency=${kafka_dependency} \ -Dkylin.rest.address=${kylin_rest_address} \ -Dspring.profiles.active=${spring_profile} \ org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.out 2>&1 & echo $! > ${KYLIN_HOME}/pid & http://git-wip-us.apache.org/repos/asf/kylin/blob/4e060e7b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index af2ed9f..a138eec 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -226,11 +226,11 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { } } - // for hive dependencies + // for kafka dependencies if (kylinKafkaDependency != null) { kylinKafkaDependency = kylinKafkaDependency.replace(":", ","); - logger.info("Kafka Dependencies Before Filtered: " + kylinHiveDependency); + logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency); if (kylinDependency.length() > 0) kylinDependency.append(",");
