This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this
push:
new 07fd1b4 KYLIN-4724 Support to download Apache Spark 2.4.6 binary
package in download-spark.sh
07fd1b4 is described below
commit 07fd1b4566ff5a84e15d32f5584b21e0861c8109
Author: yaqian.zhang <598593...@qq.com>
AuthorDate: Thu Aug 27 18:37:26 2020 +0800
KYLIN-4724 Support to download Apache Spark 2.4.6 binary package in
download-spark.sh
---
build/bin/download-spark.sh | 16 ++++-----
build/script/download-spark.sh | 75 ------------------------------------------
2 files changed, 7 insertions(+), 84 deletions(-)
diff --git a/build/bin/download-spark.sh b/build/bin/download-spark.sh
index f95e905..906a23a 100755
--- a/build/bin/download-spark.sh
+++ b/build/bin/download-spark.sh
@@ -36,21 +36,19 @@ if [[ `uname -a` =~ "Darwin" ]]; then
alias md5cmd="md5 -q"
fi
-spark_version="2.4.1"
-spark_pkg_name="spark-2.4.1-os-kylin-r3"
-spark_pkg_md5="8fb09dbb61f26f5679be49c2c8713da3"
-spark_pkg_file_name="${spark_pkg_name}.tgz"
+spark_version="2.4.6"
+spark_pkg_md5="82364f8765d03dfb14cb9c606d678058"
if [ ! -f "spark-${spark_version}-bin-hadoop2.7.tgz" ]
then
echo "No binary file found, start to download package to
${spark_package_dir}"
- wget
https://download-resource.s3.cn-north-1.amazonaws.com.cn/osspark/${spark_pkg_file_name}||
echo "Download spark failed"
+ wget
http://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop2.7.tgz||
echo "Download spark failed"
else
if [ `md5cmd spark-${spark_version}-bin-hadoop2.7.tgz | awk '{print $1}'`
!= "${spark_pkg_md5}" ]
then
echo "md5 check failed"
- rm ${spark_pkg_file_name}
- wget
https://download-resource.s3.cn-north-1.amazonaws.com.cn/osspark/${spark_pkg_file_name}||
echo "Download spark failed"
+ rm spark-${spark_version}-bin-hadoop2.7.tgz
+ wget
http://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop2.7.tgz
|| echo "Download spark failed"
else
echo "Spark package found in ${spark_package_dir}"
fi
@@ -58,8 +56,8 @@ fi
unalias md5cmd
echo "Start to decompress package"
-tar -zxvf ${spark_pkg_file_name} || { exit 1; }
-mv ${spark_pkg_name} spark
+tar -zxvf spark-${spark_version}-bin-hadoop2.7.tgz || { exit 1; }
+mv spark-${spark_version}-bin-hadoop2.7.tgz spark
# Remove unused components in Spark
rm -rf spark/lib/spark-examples-*
diff --git a/build/script/download-spark.sh b/build/script/download-spark.sh
deleted file mode 100755
index 547ea42..0000000
--- a/build/script/download-spark.sh
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-
-if [ -d "${KYLIN_HOME}/spark" ]
-then
- echo "Spark binary exists"
- exit 0;
-else
- echo "Downloading spark package..."
-fi
-
-spark_package_dir=/tmp/spark_package
-
-mkdir -p -- "${spark_package_dir}" && cd -P -- "${spark_package_dir}"
-
-alias md5cmd="md5sum"
-if [[ `uname -a` =~ "Darwin" ]]; then
- alias md5cmd="md5 -q"
-fi
-
-spark_version="2.4.1"
-spark_pkg_name="spark-2.4.1-os-kylin-r3"
-spark_pkg_md5="8fb09dbb61f26f5679be49c2c8713da3"
-spark_pkg_file_name="${spark_pkg_name}.tgz"
-
-if [ ! -f "spark-${spark_version}-bin-hadoop2.7.tgz" ]
-then
- echo "No binary file found, start to download package to
${spark_package_dir}"
- wget
https://download-resource.s3.cn-north-1.amazonaws.com.cn/osspark/${spark_pkg_file_name}||
echo "Download spark failed"
-else
- if [ `md5cmd spark-${spark_version}-bin-hadoop2.7.tgz | awk '{print $1}'`
!= "${spark_pkg_md5}" ]
- then
- echo "md5 check failed"
- rm ${spark_pkg_file_name}
- wget
https://download-resource.s3.cn-north-1.amazonaws.com.cn/osspark/${spark_pkg_file_name}||
echo "Download spark failed"
- else
- echo "Spark package found in ${spark_package_dir}"
- fi
-fi
-unalias md5cmd
-
-echo "Start to decompress package"
-tar -zxvf spark-${spark_version}-bin-hadoop2.7.tgz || { exit 1; }
-mv spark-${spark_version}-bin-hadoop2.7 spark
-
-# Remove unused components in Spark
-rm -rf spark/lib/spark-examples-*
-rm -rf spark/examples
-rm -rf spark/data
-rm -rf spark/R
-
-# mv spark binary to KYLIN_HOME
-mv spark ${KYLIN_HOME}
-
-echo "Download spark binary done"
-
-rm -rf ${spark_package_dir}
