This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.6.x by this push:
new 02a4216 KYLIN-3571 Not build Spark in Kylin's binary package
02a4216 is described below
commit 02a42161e7aab97c395232d12a4010bd9cdadf31
Author: chao long <wayn...@qq.com>
AuthorDate: Fri Mar 8 15:55:05 2019 +0800
KYLIN-3571 Not build Spark in Kylin's binary package
---
build/bin/download-spark.sh | 73 +++++++++++++++++++++++++++++++++++++++++++++
build/script/compress.sh | 6 ++--
build/script/package.sh | 1 -
3 files changed, 76 insertions(+), 4 deletions(-)
diff --git a/build/bin/download-spark.sh b/build/bin/download-spark.sh
new file mode 100755
index 0000000..4ce5f8e
--- /dev/null
+++ b/build/bin/download-spark.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
+if [ -d "${KYLIN_HOME}/spark" ]
+then
+ echo "Spark binary exists"
+ exit 0;
+else
+ echo "Downloading spark package..."
+fi
+
+spark_package_dir=/tmp/spark_package
+
+mkdir -p -- "${spark_package_dir}" && cd -P -- "${spark_package_dir}"
+
+alias md5cmd="md5sum"
+if [[ `uname -a` =~ "Darwin" ]]; then
+ alias md5cmd="md5 -q"
+fi
+
+spark_version="2.3.2"
+spark_pkg_md5="01b3e6aef0ef574e066d15e035b9b2dd"
+
+if [ ! -f "spark-${spark_version}-bin-hadoop2.7.tgz" ]
+then
+ echo "No binary file found, start to download package to
${spark_package_dir}"
+ wget
http://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop2.7.tgz
|| echo "Download spark failed"
+else
+ if [ `md5cmd spark-${spark_version}-bin-hadoop2.7.tgz | awk '{print $1}'`
!= "${spark_pkg_md5}" ]
+ then
+ echo "md5 check failed"
+ rm spark-${spark_version}-bin-hadoop2.7.tgz
+ wget
http://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop2.7.tgz
|| echo "Download spark failed"
+ else
+ echo "Spark package found in ${spark_package_dir}"
+ fi
+fi
+unalias md5cmd
+
+echo "Start to decompress package"
+tar -zxvf spark-${spark_version}-bin-hadoop2.7.tgz || { exit 1; }
+mv spark-${spark_version}-bin-hadoop2.7 spark
+
+# Remove unused components in Spark
+rm -rf spark/lib/spark-examples-*
+rm -rf spark/examples
+rm -rf spark/data
+rm -rf spark/R
+
+# mv spark binary to KYLIN_HOME
+mv spark ${KYLIN_HOME}
+
+echo "Download spark binary done"
+
+rm -rf ${spark_package_dir}
diff --git a/build/script/compress.sh b/build/script/compress.sh
index cc94f47..666dcec 100755
--- a/build/script/compress.sh
+++ b/build/script/compress.sh
@@ -34,8 +34,8 @@ package_name=apache-kylin-${version}-bin
cd build/
rm -rf ${package_name}
mkdir ${package_name}
-cp -r ext lib tool bin conf tomcat spark ../examples/sample_cube commit_SHA1
${package_name}
-rm -rf ext lib tomcat spark commit_SHA1
+cp -r ext lib tool bin conf tomcat ../examples/sample_cube commit_SHA1
${package_name}
+rm -rf ext lib tomcat commit_SHA1
## comment all default properties, and append them to the user visible
kylin.properties
## first 16 lines are license, just skip them
@@ -44,7 +44,7 @@ sed '1,16d'
../core-common/src/main/resources/kylin-defaults.properties | awk '{
find ${package_name} -type d -exec chmod 755 {} \;
find ${package_name} -type f -exec chmod 644 {} \;
find ${package_name} -type f -name "*.sh" -exec chmod 755 {} \;
-find ${package_name}/spark/bin/ -type f -exec chmod +x {} \;
+
mkdir -p ../dist
tar -cvzf ../dist/${package_name}.tar.gz ${package_name}
rm -rf ${package_name}
diff --git a/build/script/package.sh b/build/script/package.sh
index dedd10d..70e6a60 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -76,6 +76,5 @@ git rev-parse HEAD >> build/commit_SHA1
sh build/script/build.sh $@ || { exit 1; }
sh build/script/download-tomcat.sh || { exit 1; }
-sh build/script/download-spark.sh || { exit 1; }
sh build/script/prepare.sh || { exit 1; }
sh build/script/compress.sh || { exit 1; }
