This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 18f6914417 KYLIN-5955 Kylin Package support gluten
18f6914417 is described below
commit 18f6914417c528b76c6b211d429223617c217e06
Author: zhong.zhu <[email protected]>
AuthorDate: Thu Aug 29 16:00:08 2024 +0800
KYLIN-5955 Kylin Package support gluten
---
build/release/compress.sh | 21 +++++++++++++++++++++
build/release/download-gluten.sh | 39 +++++++++++++++++++++++++++++++++++++++
build/release/package.sh | 6 ++++++
build/release/release.sh | 5 +++++
build/sbin/bootstrap.sh | 2 +-
build/sbin/download-spark-user.sh | 14 +++++++++++---
6 files changed, 83 insertions(+), 4 deletions(-)
diff --git a/build/release/compress.sh b/build/release/compress.sh
index 25bbd20c21..3f887fafb6 100755
--- a/build/release/compress.sh
+++ b/build/release/compress.sh
@@ -93,6 +93,27 @@ rm -rf ext lib commit_SHA1 VERSION # keep the spark folder
on purpose
cp -rf server/webapp/dist ${package_name}/server/public
cp -rf server/newten.jar ${package_name}/server/
cp -rf server/jars ${package_name}/server/
+echo $WITH_GLUTEN
+if [[ "${WITH_GLUTEN}" = "1" ]]; then
+ mkdir -p ${package_name}/lib/gluten/
+ cp -rf gluten/jars/spark33/* ${package_name}/lib/gluten/
+ mv spark/libch.so ${package_name}/server/
+ cp spark/jars/gluten.jar ${package_name}/lib/ext/
+ if [[ "$(uname)" == "Darwin" ]]; then
+ sed -i '' '27a\
+ export LD_PRELOAD=${KYLIN_HOME}/server/libch.so
+ ' ${package_name}/sbin/bootstrap.sh
+ else
+ sed -i '27aexport LD_PRELOAD=${KYLIN_HOME}/server/libch.so'
${package_name}/sbin/bootstrap.sh
+ fi
+ else
+ if [[ "$(uname)" == "Darwin" ]]; then
+ sed -i '' '/^export LD_PRELOAD=/d'
${package_name}/sbin/bootstrap.sh
+ else
+ sed -i '/^export LD_PRELOAD=/d' ${package_name}/sbin/bootstrap.sh
+ fi
+fi
+
# cp -rf deploy/.keystore ${package_name}/server/
# mv ${package_name}/server/jars/log4j* ${package_name}/spark/jars/
rm -rf server/
diff --git a/build/release/download-gluten.sh b/build/release/download-gluten.sh
new file mode 100755
index 0000000000..d3e708799f
--- /dev/null
+++ b/build/release/download-gluten.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}/../..
+
+source build/release/functions.sh
+
+rm -rf build/gluten
+
+gluten_version=$(sed -n 's:.*<gluten.version>\(.*\)</gluten.version>.*:\1:p'
pom.xml)
+# optional
+gluten_platform='ubuntu22.04-x86_64'
+if [ ! -f "build/gluten-${gluten_version}-${gluten_platform}.tar.gz" ]; then
+ echo "no binary file found"
+ wget --directory-prefix=build/
https://repo-ofs.kyligence.com/repository/raw-tars-hosted/org/apache/gluten/${gluten_version}-${gluten_platform}/gluten-${gluten_version}-${gluten_platform}.tar.gz
+fi
+
+tar -zxf build/gluten-${gluten_version}-${gluten_platform}.tar.gz -C build/
+mv build/gluten-${gluten_version}-${gluten_platform} build/gluten
+cp build/gluten/libs/libch.so build/spark/
+find build/spark/jars/ -name "protobuf-java*" -delete
+cp -r build/gluten/jars/spark33/* build/spark/jars/
diff --git a/build/release/package.sh b/build/release/package.sh
index 22f227977d..ab978b053c 100755
--- a/build/release/package.sh
+++ b/build/release/package.sh
@@ -55,6 +55,12 @@ bash build/release/build.sh $@ || { exit 1; }
if [[ "${WITH_SPARK}" = "1" ]]; then
echo "BUILD STAGE 3 - Prepare spark..."
bash -x build/release/download-spark.sh || { exit 1; }
+ if [[ "${WITH_GLUTEN}" = "1" ]]; then
+ echo "Prepare gluten..."
+ sh build/release/download-gluten.sh || { exit 1; }
+ else
+ sed -i '/^export LD_PRELOAD=/d' build/sbin/bootstrap.sh
+ fi
else
rm -rf build/spark
fi
diff --git a/build/release/release.sh b/build/release/release.sh
index b2277becb6..18d4d8a5e1 100755
--- a/build/release/release.sh
+++ b/build/release/release.sh
@@ -64,6 +64,11 @@ for PARAM in $@; do
export SKIP_COMPILE=1
shift
fi
+ if [[ "$PARAM" == "-withGluten" ]]; then
+ echo 'Package with Gluten relevant dependencies...'
+ export WITH_GLUTEN=1
+ shift
+ fi
done
## Fetch kylin version from pom
diff --git a/build/sbin/bootstrap.sh b/build/sbin/bootstrap.sh
index 60cd8fb8e4..7b50399ec4 100755
--- a/build/sbin/bootstrap.sh
+++ b/build/sbin/bootstrap.sh
@@ -204,7 +204,7 @@ function runToolInternal() {
else
kylin_tools_log4j="file:${KYLIN_HOME}/tool/conf/kylin-tools-log4j.xml"
fi
- java -Xms${JAVA_VM_TOOL_XMS} -Xmx${JAVA_VM_TOOL_XMX}
${KYLIN_KERBEROS_OPTS} -Dfile.encoding=UTF-8
-Dlog4j.configurationFile=${kylin_tools_log4j}
-Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} -Dhdp.version=current -cp
"${kylin_hadoop_conf_dir}:${KYLIN_HOME}/conf/:${KYLIN_HOME}/lib/ext/*:${KYLIN_HOME}/server/jars/*:${SPARK_HOME}/jars/*"
"$@"
+ java -Xms${JAVA_VM_TOOL_XMS} -Xmx${JAVA_VM_TOOL_XMX}
${KYLIN_KERBEROS_OPTS} -Dfile.encoding=UTF-8
-Dlog4j.configurationFile=${kylin_tools_log4j}
-Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} -Dhdp.version=current -cp
"${kylin_hadoop_conf_dir}:${KYLIN_HOME}/conf/:${KYLIN_HOME}/lib/ext/*:${SPARK_HOME}/jars/*:${KYLIN_HOME}/server/jars/*"
"$@"
}
function killChildProcess {
diff --git a/build/sbin/download-spark-user.sh
b/build/sbin/download-spark-user.sh
index 12a7e1bced..6161f6780f 100755
--- a/build/sbin/download-spark-user.sh
+++ b/build/sbin/download-spark-user.sh
@@ -41,8 +41,8 @@ echo "spark_pkg_file_name : "${spark_pkg_file_name}
wget --directory-prefix=${KYLIN_HOME}
https://s3.cn-north-1.amazonaws.com.cn/download-resource/kyspark/${spark_pkg_file_name}
|| echo "Download spark failed"
mkdir -p ${KYLIN_HOME}/${spark_pkg_name}
-tar -zxf ${spark_pkg_file_name} -C ${spark_pkg_name} --strip-components 1 || {
exit 1; }
-mv ${spark_pkg_name} spark
+tar -zxf ${KYLIN_HOME}/${spark_pkg_file_name} -C
${KYLIN_HOME}/${spark_pkg_name} --strip-components 1 || { exit 1; }
+mv ${KYLIN_HOME}/${spark_pkg_name} ${KYLIN_HOME}/spark
# Remove unused components in Spark
rm -rf ${KYLIN_HOME}/spark/lib/spark-examples-*
@@ -73,4 +73,12 @@ else
wget --directory-prefix=${KYLIN_HOME}
https://s3.cn-north-1.amazonaws.com.cn/download-resource/kyspark/hive_1_2_2.tar.gz
|| echo "Download hive1 failed"
fi
fi
-tar -zxf hive_1_2_2.tar.gz -C ${KYLIN_HOME}/spark/ || { exit 1; }
+tar -zxf ${KYLIN_HOME}/hive_1_2_2.tar.gz -C ${KYLIN_HOME}/spark/ || { exit 1; }
+
+# add gluten relevant dependencies to spark
+if [ ! -f ${KYLIN_HOME}/lib/gluten ]
+then
+ find ${KYLIN_HOME}/spark/jars/ -name "protobuf-java*" -delete
+ cp -rf ${KYLIN_HOME}/server/jars/*.jar ${KYLIN_HOME}/spark/jars/
+ cp -rf ${KYLIN_HOME}/server/libch.so ${KYLIN_HOME}/spark/
+fi
\ No newline at end of file
