This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ef1e7cc37af [opt](deps) add jindofs in classpath after hadoop libs
(#51689)
ef1e7cc37af is described below
commit ef1e7cc37af283f8a571a14f96785b9640398d67
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Sat Jun 14 15:13:59 2025 +0800
[opt](deps) add jindofs in classpath after hadoop libs (#51689)
### What problem does this PR solve?
Related PR: #49903
Problem Summary:
We should add jindofs jars in CLASSPATH, but after hadoop jars.
So that we can use the right hadoop deps instead of deps from jindofs.
---
bin/start_be.sh | 8 ++++++++
bin/start_fe.sh | 8 ++++++++
build.sh | 24 +++++++++++++-----------
3 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/bin/start_be.sh b/bin/start_be.sh
index 391885ecde7..db553ecb8d2 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -243,6 +243,14 @@ if [[ -d "${DORIS_HOME}/lib/hadoop_hdfs/" ]]; then
done
fi
+# add jindofs
+# should after jars in lib/hadoop_hdfs/, or it will override the hadoop jars
in lib/hadoop_hdfs
+if [[ -d "${DORIS_HOME}/lib/java_extensions/jindofs" ]]; then
+ for f in "${DORIS_HOME}/lib/java_extensions/jindofs"/*.jar; do
+ DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
+ done
+fi
+
# add custom_libs to CLASSPATH
# ATTN, custom_libs is deprecated, use plugins/java_extensions
if [[ -d "${DORIS_HOME}/custom_lib" ]]; then
diff --git a/bin/start_fe.sh b/bin/start_fe.sh
index 4a728787a82..2fd0920df63 100755
--- a/bin/start_fe.sh
+++ b/bin/start_fe.sh
@@ -224,6 +224,14 @@ if [[ -d "${DORIS_HOME}/custom_lib" ]]; then
done
fi
+# add jindofs
+# should after jars in lib/, or it will override the hadoop jars in lib/
+if [[ -d "${DORIS_HOME}/lib/jindofs" ]]; then
+ for f in "${DORIS_HOME}/lib/jindofs"/*.jar; do
+ CLASSPATH="${CLASSPATH}:${f}"
+ done
+fi
+
# add plugins/java_extensions to CLASSPATH
if [[ -d "${DORIS_HOME}/plugins/java_extensions" ]]; then
for f in "${DORIS_HOME}/plugins/java_extensions"/*.jar; do
diff --git a/build.sh b/build.sh
index 0a4bed4135b..c24e24d3ce4 100755
--- a/build.sh
+++ b/build.sh
@@ -755,18 +755,19 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
cp -r -p "${DORIS_HOME}/conf/ldap.conf" "${DORIS_OUTPUT}/fe/conf"/
cp -r -p "${DORIS_HOME}/conf/mysql_ssl_default_certificate"
"${DORIS_OUTPUT}/fe/"/
rm -rf "${DORIS_OUTPUT}/fe/lib"/*
+ install -d "${DORIS_OUTPUT}/fe/lib/jindofs"
cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/
cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar"
"${DORIS_OUTPUT}/fe/lib"/
#cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip"
"${DORIS_OUTPUT}/fe/lib"/
# copy jindofs jars, only support for Linux x64 or arm
if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'x86_64'
]]; then
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" ==
'aarch64' ]]; then
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/fe/lib/jindofs"/
fi
cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/
@@ -799,7 +800,7 @@ if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
"${DORIS_OUTPUT}/be/www" \
"${DORIS_OUTPUT}/be/tools/FlameGraph"
- cp -r -p "${DORIS_HOME}/be/output/bin"/* "${DORIS_OUTPUT}/be/bin"/
+ cp -r -p "${DORIS_HOME}/bin"/*_be.sh "${DORIS_OUTPUT}/be/bin"/
cp -r -p "${DORIS_HOME}/be/output/conf"/* "${DORIS_OUTPUT}/be/conf"/
cp -r -p "${DORIS_HOME}/be/output/dict" "${DORIS_OUTPUT}/be/"
@@ -907,13 +908,14 @@ EOF
done
# copy jindofs jars, only support for Linux x64 or arm
+ install -d "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'x86_64'
]]; then
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'aarch64'
]]; then
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
- cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
+ cp -r -p
"${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar
"${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
fi
cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/*
"${DORIS_OUTPUT}/be/www"/
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
