KYLIN-2605 Execute sample.sh script failure in hdfs federation environment Signed-off-by: Billy Liu <billy...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d449c40d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d449c40d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d449c40d Branch: refs/heads/KYLIN-2624 Commit: d449c40d58b8af10ebc638be4aadb80e09787a9d Parents: b3310dc Author: 10069681 <peng.jian...@zte.com.cn> Authored: Thu May 11 11:19:01 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Thu May 11 13:34:24 2017 +0800 ---------------------------------------------------------------------- build/bin/sample.sh | 25 ++++++++++++++++------ examples/sample_cube/create_sample_tables.sql | 10 ++++----- 2 files changed, 23 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d449c40d/build/bin/sample.sh ---------------------------------------------------------------------- diff --git a/build/bin/sample.sh b/build/bin/sample.sh index 617b671..10900bb 100644 --- a/build/bin/sample.sh +++ b/build/bin/sample.sh @@ -31,9 +31,20 @@ else hadoop_conf_param="--config ${kylin_hadoop_conf_dir}" fi -echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data" -hadoop ${hadoop_conf_param} fs -mkdir -p /tmp/kylin/sample_cube/data -hadoop ${hadoop_conf_param} fs -put * /tmp/kylin/sample_cube/data/ +if [ -z "$1" ]; then + hdfs_tmp_dir=/tmp/kylin +else + hdfs_tmp_dir=$1 +fi + +echo "Loading sample data into HDFS tmp path: ${hdfs_tmp_dir}/sample_cube/data" +hadoop ${hadoop_conf_param} fs -mkdir -p ${hdfs_tmp_dir}/sample_cube/data +if [ $? != 0 ] +then + quit "Failed to create ${hdfs_tmp_dir}/sample_cube/data. Please make sure the user has right to access ${hdfs_tmp_dir}/sample_cube/data or usage: sample.sh hdfs_tmp_dir" +fi + +hadoop ${hadoop_conf_param} fs -put * ${hdfs_tmp_dir}/sample_cube/data/ hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client` sample_database=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.database-for-flat-table` @@ -51,14 +62,14 @@ then else beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}} fi - beeline ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } + beeline ${beeline_params} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } else hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database - hive --database $sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } + hive --database $sample_database --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } fi echo "Sample hive tables are created successfully; Going to create sample cube..." -hadoop ${hadoop_conf_param} fs -rm -r /tmp/kylin/sample_cube +hadoop ${hadoop_conf_param} fs -rm -r ${hdfs_tmp_dir}/sample_cube # set engine type and storage type to cube desc default_engine_type=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.engine.default` @@ -85,4 +96,4 @@ ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv & $sample_database\ cd ${KYLIN_HOME} hbase org.apache.hadoop.util.RunJar ${job_jar} org.apache.kylin.common.persistence.ResourceTool upload ${KYLIN_HOME}/sample_cube/metadata || { exit 1; } echo "Sample cube is created successfully in project 'learn_kylin'." -echo "Restart Kylin server or reload the metadata from web UI to see the change." +echo "Restart Kylin server or reload the metadata from web UI to see the change." \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d449c40d/examples/sample_cube/create_sample_tables.sql ---------------------------------------------------------------------- diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql index 24f26f8..c33a99e 100644 --- a/examples/sample_cube/create_sample_tables.sql +++ b/examples/sample_cube/create_sample_tables.sql @@ -216,8 +216,8 @@ COMMENT 'Sales order table, fact table' ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE; -LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE KYLIN_SALES; -LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO TABLE KYLIN_ACCOUNT; -LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO TABLE KYLIN_COUNTRY; -LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE KYLIN_CAL_DT; -LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE KYLIN_CATEGORY_GROUPINGS; +LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE KYLIN_SALES; +LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO TABLE KYLIN_ACCOUNT; +LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO TABLE KYLIN_COUNTRY; +LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE KYLIN_CAL_DT; +LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE KYLIN_CATEGORY_GROUPINGS;
