This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 11e5951657 KYLIN-5972 Support start up without gluten
11e5951657 is described below
commit 11e59516574fa6f6598787650d4b3a003c55c4a1
Author: zhong.zhu <[email protected]>
AuthorDate: Tue Sep 10 19:50:49 2024 +0800
KYLIN-5972 Support start up without gluten
---
build/bin/kylin.sh | 15 ++++++-----
build/conf/kylin.properties.withoutGluten | 30 +++++++++++++++++++++
build/release/compress.sh | 1 +
build/sbin/bootstrap.sh | 45 +++++++++++++++++++++++++++++++
4 files changed, 85 insertions(+), 6 deletions(-)
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 8fe5e8b011..9bfada4096 100755
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -23,25 +23,29 @@ function help() {
echo "Commands:"
echo " -D -D[ skipCheck ]"
echo " skip some check when bootstrap, eg. bash
kylin.sh -DskipCheck start"
+ echo " -D[ withoutGluten ]"
+ echo " skip skip load gluten dependencies, eg. bash
kylin.sh -DwithoutGluten start"
echo " -C -C[ true | false ], default true, use the
local properties or not"
echo " start start kylin"
echo " restart restart kylin"
echo " stop stop kylin"
- echo " io.kyligence.* run tool"
+ echo " org.apache.kylin.* run tool"
echo " interactive Enter for bootstrap"
}
-function parseSkipCheckArgs() {
+function parseArgs() {
case $1 in
"skipCheck")
- skipValue=1
+ export KYLIN_SKIP_CHECK=1
+ ;;
+ "withoutGluten")
+ export KYLIN_WITH_GLUTEN=0
;;
*)
echo "Invalid option -D: -$1" >&2
exit 1
;;
esac
- echo ${skipValue}
}
function checkArguments() {
@@ -78,8 +82,7 @@ function main() {
export verbose=true
;;
D)
- skipArgs=$(parseSkipCheckArgs "$OPTARG") || exit 1
- export KYLIN_SKIP_CHECK=${skipArgs}
+ parseArgs "$OPTARG" || exit 1
;;
C)
export _KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE="$OPTARG"
diff --git a/build/conf/kylin.properties.withoutGluten
b/build/conf/kylin.properties.withoutGluten
new file mode 100644
index 0000000000..bbbbc98e44
--- /dev/null
+++ b/build/conf/kylin.properties.withoutGluten
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Kylin provides configurations to start withoutGluten
+# If you want to start kylin without gluten, please run `kylin.sh
-DwithoutGluten start`
+kylin.storage.columnar.spark-conf.spark.gluten.enabled=false
+kylin.storage.columnar.spark-conf.spark.plugins=
+kylin.storage.columnar.spark-conf.spark.memory.offHeap.enabled=false
+kylin.storage.columnar.spark-conf.spark.shuffle.manager=org.apache.spark.shuffle.sort.SortShuffleManager
+kylin.storage.columnar.spark-conf.spark.executorEnv.LD_PRELOAD=
+
+kylin.engine.spark-conf.spark.gluten.enabled=false
+kylin.engine.spark-conf.spark.plugins=
+kylin.engine.spark-conf.spark.memory.offHeap.enabled=false
+kylin.engine.spark-conf.spark.shuffle.manager=org.apache.spark.shuffle.sort.SortShuffleManager
+kylin.engine.spark-conf.spark.executorEnv.LD_PRELOAD=
\ No newline at end of file
diff --git a/build/release/compress.sh b/build/release/compress.sh
index 6a57f53660..5fe084218a 100755
--- a/build/release/compress.sh
+++ b/build/release/compress.sh
@@ -81,6 +81,7 @@ for log_conf in `find conf -name "*-log4j.xml"`; do
done
cp -rf conf/kylin.properties ${package_name}/conf/kylin.properties
cp -rf conf/setenv.sh ${package_name}/conf/setenv.sh.template
+cp -rf conf/kylin.properties.withoutGluten ${package_name}/conf/
cp -rf bin/ ${package_name}/bin/
cp -rf sbin/ ${package_name}/sbin/
diff --git a/build/sbin/bootstrap.sh b/build/sbin/bootstrap.sh
index 7db4cc2941..770ae4de94 100755
--- a/build/sbin/bootstrap.sh
+++ b/build/sbin/bootstrap.sh
@@ -24,6 +24,7 @@ version=$(cat "${KYLIN_HOME}"/VERSION | awk '{print $3}')
export KYLIN_SKIP_CHECK=${KYLIN_SKIP_CHECK:-0}
KYLIN_SKIP_CHECK_MODE=1
+export KYLIN_WITH_GLUTEN=${KYLIN_WITH_GLUTEN:-1}
if [ "$1" == "-v" ]; then
shift
@@ -68,6 +69,50 @@ function prepareEnv() {
verbose "LD_PRELOAD= is:${LD_PRELOAD}"
fi
+ # check if start with gluten
+ if [[ ${KYLIN_WITH_GLUTEN} -eq 0 ]]; then
+ verbose "start without gluten"
+ if [[ -f ${KYLIN_HOME}/lib/ext/gluten.jar ]]; then
+ mv "${KYLIN_HOME}"/lib/ext/gluten.jar
"${KYLIN_HOME}"/lib/ext/gluten.jar.bak
+ fi
+
+ if [[ -f ${KYLIN_HOME}/spark/jars/gluten.jar ]]; then
+ mv "${KYLIN_HOME}"/spark/jars/gluten.jar
"${KYLIN_HOME}"/spark/jars/gluten.jar.bak
+ fi
+
+ gluten_shims_jar=$(find "${KYLIN_HOME}"/server/jars -name
"spark-sql-columnar-shims-kyspark-*.jar")
+ if [[ -f "$gluten_shims_jar" ]]; then
+ mv "$gluten_shims_jar" "$gluten_shims_jar".bak
+ fi
+
+ config_without_gluten=$(grep -vE "^#"
"${KYLIN_HOME}/conf/kylin.properties.withoutGluten" | grep -E "^kylin.*")
+ while read -r line
+ do
+ sed -i "/${line}/d" "${KYLIN_CONFIG_FILE}"
+ done <<< "$config_without_gluten"
+ echo "$config_without_gluten" >> "${KYLIN_CONFIG_FILE}"
+ else
+ verbose "start with gluten"
+ if [[ -f ${KYLIN_HOME}/lib/ext/gluten.jar.bak ]]; then
+ mv "${KYLIN_HOME}"/lib/ext/gluten.jar.bak
"${KYLIN_HOME}"/lib/ext/gluten.jar
+ fi
+
+ if [[ -f ${KYLIN_HOME}/spark/jars/gluten.jar.bak ]]; then
+ mv "${KYLIN_HOME}"/spark/jars/gluten.jar.bak
"${KYLIN_HOME}"/spark/jars/gluten.jar
+ fi
+
+ gluten_shims_jar=$(find "${KYLIN_HOME}"/server/jars -name
"spark-sql-columnar-shims-kyspark-*.jar.bak")
+ if [[ -f "$gluten_shims_jar" ]]; then
+ mv "$gluten_shims_jar" "${gluten_shims_jar%.bak}"
+ fi
+
+ config_without_gluten=$(grep -vE "^#"
"${KYLIN_HOME}/conf/kylin.properties.withoutGluten" | grep -E "^kylin.*")
+ while read -r line
+ do
+ sed -i "/${line}/d" "${KYLIN_CONFIG_FILE}"
+ done <<< "$config_without_gluten"
+ fi
+
}
function retrieveDependency() {
