This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new ea30267ac6f branch-4.1: [fix](thirdparty) order Paimon codec
dependencies #67158 (#67357)
ea30267ac6f is described below
commit ea30267ac6f9ac3f09436d91e5dd9976c4173ec0
Author: Gabriel <[email protected]>
AuthorDate: Wed Sep 2 06:15:55 2026 +0800
branch-4.1: [fix](thirdparty) order Paimon codec dependencies #67158
(#67357)
### What problem does this PR solve?
Backport #67158 to branch-4.1.
A clean parallel Paimon C++ build can start compiling the object library
before bundled codec external projects install their headers. External
Regression can also replace the packaged FE configuration without
preserving the compatible ADBC JNI library path.
### What is changed and how does it work?
- Add the zstd, snappy, and lz4 targets to the Paimon object library
dependencies.
- Keep compatible complete Paimon prebuilts reusable.
- Preserve the ADBC JNI library path in External Regression.
- Add focused regression checks to the Arrow/Paimon workflow.
### Check List
- [x] Arrow/Paimon lifecycle test
- [x] Paimon codec dependency regression test
- [x] ADBC JNI configuration regression test
- [x] Patch dry-run against paimon-cpp-0a4f4e2
- [x] Shell syntax and workflow YAML validation
### Release note
None.
---
.github/workflows/build-thirdparty.yml | 8 ++-
regression-test/pipeline/external/conf/fe.conf | 3 +-
thirdparty/arrow-paimon-vars.sh | 4 +-
.../paimon-cpp-buildutils-static-deps.patch | 17 +++++
thirdparty/test/adbc-jni-config-test.sh | 35 ++++++++++
thirdparty/test/paimon-codec-dependency-test.sh | 78 ++++++++++++++++++++++
6 files changed, 142 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/build-thirdparty.yml
b/.github/workflows/build-thirdparty.yml
index 7144cfe8425..9a780f8f04c 100644
--- a/.github/workflows/build-thirdparty.yml
+++ b/.github/workflows/build-thirdparty.yml
@@ -65,7 +65,10 @@ jobs:
- 'thirdparty/paimon-cpp-cache.cmake'
- 'thirdparty/patches/apache-arrow-*.patch'
- 'thirdparty/patches/paimon-cpp-*.patch'
+ - 'regression-test/pipeline/external/conf/fe.conf'
- 'thirdparty/test/arrow-paimon-lifecycle-test.sh'
+ - 'thirdparty/test/adbc-jni-config-test.sh'
+ - 'thirdparty/test/paimon-codec-dependency-test.sh'
- '.github/workflows/build-thirdparty.yml'
arrow_paimon_lifecycle_test:
@@ -78,7 +81,10 @@ jobs:
uses: actions/checkout@v4
- name: Test focused thirdparty lifecycle
- run: thirdparty/test/arrow-paimon-lifecycle-test.sh
+ run: |
+ thirdparty/test/arrow-paimon-lifecycle-test.sh
+ thirdparty/test/adbc-jni-config-test.sh
+ thirdparty/test/paimon-codec-dependency-test.sh
build_linux:
name: Build Third Party Libraries (Linux)
diff --git a/regression-test/pipeline/external/conf/fe.conf
b/regression-test/pipeline/external/conf/fe.conf
index fc818a862a1..4466998ae67 100644
--- a/regression-test/pipeline/external/conf/fe.conf
+++ b/regression-test/pipeline/external/conf/fe.conf
@@ -28,7 +28,8 @@ DATE = `date +%Y%m%d-%H%M%S`
JAVA_OPTS="-Xmx4096m -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=$DORIS_HOME/log/fe.jmap -XX:+UseMembar -XX:SurvivorRatio=8
-XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+PrintClassHistogramAfterFullGC -XX:+UseConcMarkSweepGC -XX:+UseParNewGC
-XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled
-XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0
-Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE
-Dcom.mysql.cj.disableAbandonedConnectionCleanup=true"
# For jdk 17+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8
-Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR
-Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNN [...]
+# External deployment replaces the packaged fe.conf, so keep Doris'
glibc-compatible ADBC JNI build.
+JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8
-Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR
-Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Darrow.enable_null_check_for_get=false
-Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.re [...]
##
## the lowercase properties are read by main program.
diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh
index b5c6d6955c6..ab139125035 100644
--- a/thirdparty/arrow-paimon-vars.sh
+++ b/thirdparty/arrow-paimon-vars.sh
@@ -58,7 +58,9 @@ PAIMON_BUILD_SCHEMA_VERSION="1"
# whole-script fingerprint. Keep these exact markers during the schema
transition;
# version and complete artifact validation are still mandatory before reuse.
ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="ab79ab0bbfbf93f9860050fb751b20fee9e40d96"
-PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="cb82e41ba46f534e611cdd52e66b53c227d49bf8"
+# Dependency ordering does not change the installed Paimon artifacts, so
existing
+# complete prebuilts remain compatible with the corrected clean-build graph.
+PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="9887cf1ec13106da4250f8d7ea931a258879089c"
ARROW_LEGACY_BUILD_FINGERPRINTS=(
9d03645dd1cded5184a8126f5c7f4a6eb9b92b53
)
diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
index fc1fe155312..213cc279f52 100644
--- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
+++ b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
@@ -343,3 +343,20 @@ diff --git a/src/paimon/format/blob/blob_format_writer.cpp
b/src/paimon/format/b
+ list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR)
+ list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR)
+ list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH)
+
+diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
+index 76dd03b..9317324 100644
+--- a/src/paimon/CMakeLists.txt
++++ b/src/paimon/CMakeLists.txt
+@@ -281,6 +281,11 @@ add_paimon_lib(paimon
+ ${PAIMON_COMMON_SRCS}
+ ${PAIMON_CORE_SRCS}
+ DEPENDENCIES
++ # Codec headers come from ExternalProject install steps, so
object
++ # compilation must not race those steps in clean parallel
builds.
++ zstd
++ snappy
++ lz4
+ arrow
+ tbb
+ glog
diff --git a/thirdparty/test/adbc-jni-config-test.sh
b/thirdparty/test/adbc-jni-config-test.sh
new file mode 100755
index 00000000000..6f2cf074e3c
--- /dev/null
+++ b/thirdparty/test/adbc-jni-config-test.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd)"
+EXTERNAL_FE_CONF="${ROOT}/regression-test/pipeline/external/conf/fe.conf"
+# shellcheck disable=SC2016 # Match the literal option before fe.conf expands
DORIS_HOME.
+JNI_LIBRARY_OPTION='-Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib'
+
+# External Regression replaces the packaged FE configuration instead of
merging it.
+# Keep the JNI override in that replacement so ADBC cannot fall back to the
jar binary.
+java_opts_assignment="$(grep '^JAVA_OPTS_FOR_JDK_17=' "${EXTERNAL_FE_CONF}" ||
true)"
+if [[ " ${java_opts_assignment} " != *" ${JNI_LIBRARY_OPTION} "* ]]; then
+ echo "FAIL: ${EXTERNAL_FE_CONF} drops the packaged ADBC JNI library path."
>&2
+ echo "Add ${JNI_LIBRARY_OPTION} to its FE JVM options." >&2
+ exit 1
+fi
+
+echo "PASS"
diff --git a/thirdparty/test/paimon-codec-dependency-test.sh
b/thirdparty/test/paimon-codec-dependency-test.sh
new file mode 100755
index 00000000000..4d3b92458b8
--- /dev/null
+++ b/thirdparty/test/paimon-codec-dependency-test.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)"
+
+fail() {
+ echo "FAIL: $*" >&2
+ exit 1
+}
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "${tmpdir}"' EXIT
+
+fixture="${tmpdir}/paimon-codec-dependency"
+focused_patch="${fixture}/codec-dependency.patch"
+mkdir -p "${fixture}/src/paimon"
+{
+ printf '%s\n' 'add_paimon_lib(paimon'
+ printf '%s\n' ' SOURCES'
+ printf '%s\n' ' ${PAIMON_COMMON_SRCS}'
+ printf '%s\n' ' ${PAIMON_CORE_SRCS}'
+ printf '%s\n' ' DEPENDENCIES'
+ printf '%s\n' ' arrow'
+ printf '%s\n' ' tbb'
+ printf '%s\n' ' glog'
+ printf '%s\n' ' fmt'
+ printf '%s\n' ' roaring_bitmap'
+ printf '%s\n' ' xxhash'
+ printf '%s\n' ' Threads::Threads'
+ printf '%s\n' ' RapidJSON'
+ printf '%s\n' ' STATIC_LINK_LIBS'
+ printf '%s\n' ' arrow)'
+} >"${fixture}/src/paimon/CMakeLists.txt"
+
+awk '
+ /^diff --git / {
+ if (found) {
+ exit
+ }
+ if ($0 == "diff --git a/src/paimon/CMakeLists.txt
b/src/paimon/CMakeLists.txt") {
+ found = 1
+ }
+ }
+ found { print }
+' "${ROOT}/patches/paimon-cpp-buildutils-static-deps.patch" >"${focused_patch}"
+[[ -s "${focused_patch}" ]] || fail "the Paimon codec dependency patch is
missing"
+(
+ cd "${fixture}"
+ patch -s -p1 <"${focused_patch}"
+) || fail "the Paimon codec dependency patch did not apply"
+
+dependencies="$(sed -n
'/^[[:space:]]*DEPENDENCIES$/,/^[[:space:]]*STATIC_LINK_LIBS$/p' \
+ "${fixture}/src/paimon/CMakeLists.txt")"
+grep -Eq '^[[:space:]]+zstd$' <<<"${dependencies}" ||
+ fail "paimon_objlib does not wait for the ZSTD headers"
+grep -Eq '^[[:space:]]+snappy$' <<<"${dependencies}" ||
+ fail "paimon_objlib does not wait for the Snappy headers"
+grep -Eq '^[[:space:]]+lz4$' <<<"${dependencies}" ||
+ fail "paimon_objlib does not wait for the LZ4 headers"
+
+echo "PASS"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
