KYLIN-1469 Find hive dependencies from jvm if not exists in environment variables
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b41c4460 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b41c4460 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b41c4460 Branch: refs/heads/1.5.x-HBase1.1.3 Commit: b41c44600f801f80aa379494bd3be9bc77d5fa65 Parents: 8a3e0e0 Author: lidongsjtu <[email protected]> Authored: Fri Mar 4 14:02:28 2016 +0800 Committer: lidongsjtu <[email protected]> Committed: Fri Mar 4 14:02:28 2016 +0800 ---------------------------------------------------------------------- .../engine/mr/common/AbstractHadoopJob.java | 22 +++++++++++++++++++- examples/test_case_data/sandbox/mapred-site.xml | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b41c4460/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index e4eee96..61983d5 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -23,7 +23,7 @@ package org.apache.kylin.engine.mr.common; * */ -import static org.apache.hadoop.util.StringUtils.formatTime; +import static org.apache.hadoop.util.StringUtils.*; import java.io.File; import java.io.IOException; @@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -217,6 +218,25 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { if (kylinDependency.length() > 0) kylinDependency.append(","); kylinDependency.append(filteredHive); + } else { + + logger.info("No hive dependency jars set in the environment, will find them from jvm:"); + + try { + String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver")); + kylinDependency.append(hiveExecJarPath).append(","); + logger.info("hive-exec jar file: " + hiveExecJarPath); + + String hiveHCatJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat")); + kylinDependency.append(hiveHCatJarPath).append(","); + logger.info("hive-catalog jar file: " + hiveHCatJarPath); + + String hiveMetaStoreJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table")); + kylinDependency.append(hiveMetaStoreJarPath).append(","); + logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath); + } catch (ClassNotFoundException e) { + logger.error("Cannot found hive dependency jars: " + e); + } } // for KylinJobMRLibDir http://git-wip-us.apache.org/repos/asf/kylin/blob/b41c4460/examples/test_case_data/sandbox/mapred-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml index 7b1a4be..18f6feb 100644 --- a/examples/test_case_data/sandbox/mapred-site.xml +++ b/examples/test_case_data/sandbox/mapred-site.xml @@ -58,7 +58,7 @@ <property> <name>mapreduce.application.classpath</name> - <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value> + <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value> </property> <property>
