This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8026894b8df [fix](thirdparty) order Paimon codec dependencies (#67158)
8026894b8df is described below
commit 8026894b8df8596e0e9f7f733e314e2858652454
Author: Gabriel <[email protected]>
AuthorDate: Fri Aug 28 20:44:45 2026 +0800
[fix](thirdparty) order Paimon codec dependencies (#67158)
### What problem does this PR solve?
A clean parallel Paimon C++ build can start compiling the object library
before bundled codec external projects install their headers. This makes
block_compression_factory.cpp fail with a missing codec header depending
on the build graph state.
### What is changed and how does it work?
- Add the zstd, snappy, and lz4 targets to the Paimon object library
dependencies so their ExternalProject install steps finish before codec
sources compile.
- Keep existing complete Paimon prebuilts compatible because dependency
ordering does not change installed artifacts.
- Add a focused regression test that applies the Doris patch and
verifies all three codec dependencies, and run it in the Arrow/Paimon
workflow.
---
.github/workflows/build-thirdparty.yml | 8 ++-
regression-test/pipeline/external/conf/fe.conf | 2 +-
thirdparty/arrow-paimon-vars.sh | 4 +-
.../paimon-cpp-buildutils-static-deps.patch | 17 +++++
thirdparty/test/adbc-jni-config-test.sh | 35 ++++++++++
thirdparty/test/paimon-codec-dependency-test.sh | 78 ++++++++++++++++++++++
6 files changed, 141 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/build-thirdparty.yml
b/.github/workflows/build-thirdparty.yml
index 7ae7cb2d43e..03244548080 100644
--- a/.github/workflows/build-thirdparty.yml
+++ b/.github/workflows/build-thirdparty.yml
@@ -58,7 +58,10 @@ jobs:
- 'thirdparty/paimon-cpp-cache.cmake'
- 'thirdparty/patches/apache-arrow-*.patch'
- 'thirdparty/patches/paimon-cpp-*.patch'
+ - 'regression-test/pipeline/external/conf/fe.conf'
- 'thirdparty/test/arrow-paimon-lifecycle-test.sh'
+ - 'thirdparty/test/adbc-jni-config-test.sh'
+ - 'thirdparty/test/paimon-codec-dependency-test.sh'
- '.github/workflows/build-thirdparty.yml'
arrow_paimon_lifecycle_test:
@@ -71,7 +74,10 @@ jobs:
uses: actions/checkout@v4
- name: Test focused thirdparty lifecycle
- run: thirdparty/test/arrow-paimon-lifecycle-test.sh
+ run: |
+ thirdparty/test/arrow-paimon-lifecycle-test.sh
+ thirdparty/test/adbc-jni-config-test.sh
+ thirdparty/test/paimon-codec-dependency-test.sh
script_test:
name: Thirdparty Script Test
diff --git a/regression-test/pipeline/external/conf/fe.conf
b/regression-test/pipeline/external/conf/fe.conf
index d5c3e734433..4466998ae67 100644
--- a/regression-test/pipeline/external/conf/fe.conf
+++ b/regression-test/pipeline/external/conf/fe.conf
@@ -28,7 +28,7 @@ DATE = `date +%Y%m%d-%H%M%S`
JAVA_OPTS="-Xmx4096m -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=$DORIS_HOME/log/fe.jmap -XX:+UseMembar -XX:SurvivorRatio=8
-XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+PrintClassHistogramAfterFullGC -XX:+UseConcMarkSweepGC -XX:+UseParNewGC
-XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled
-XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0
-Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE
-Dcom.mysql.cj.disableAbandonedConnectionCleanup=true"
# For jdk 17+, this JAVA_OPTS will be used as default JVM options
-# Use Doris' compatible ADBC JNI build because the Maven-bundled library
requires a newer glibc.
+# External deployment replaces the packaged fe.conf, so keep Doris'
glibc-compatible ADBC JNI build.
JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8
-Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR
-Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Darrow.enable_null_check_for_get=false
-Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.re [...]
##
diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh
index e6f8cfb7c6f..d532a740e63 100644
--- a/thirdparty/arrow-paimon-vars.sh
+++ b/thirdparty/arrow-paimon-vars.sh
@@ -61,7 +61,9 @@ PAIMON_BUILD_SCHEMA_VERSION="1"
# mandatory before reuse. Delete the lists once master has published a
prebuilt that
# carries the semantic marker itself.
ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="ab79ab0bbfbf93f9860050fb751b20fee9e40d96"
-PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="cb82e41ba46f534e611cdd52e66b53c227d49bf8"
+# Dependency ordering does not change the installed Paimon artifacts, so
existing
+# complete prebuilts remain compatible with the corrected clean-build graph.
+PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="9887cf1ec13106da4250f8d7ea931a258879089c"
ARROW_LEGACY_BUILD_FINGERPRINTS=(
# published 2026-08-19, master after apache/doris#66842
8356a3a464b93f3a59a6bdae6a51258947ab35d2
diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
index fc1fe155312..213cc279f52 100644
--- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
+++ b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
@@ -343,3 +343,20 @@ diff --git a/src/paimon/format/blob/blob_format_writer.cpp
b/src/paimon/format/b
+ list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR)
+ list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR)
+ list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH)
+
+diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
+index 76dd03b..9317324 100644
+--- a/src/paimon/CMakeLists.txt
++++ b/src/paimon/CMakeLists.txt
+@@ -281,6 +281,11 @@ add_paimon_lib(paimon
+ ${PAIMON_COMMON_SRCS}
+ ${PAIMON_CORE_SRCS}
+ DEPENDENCIES
++ # Codec headers come from ExternalProject install steps, so
object
++ # compilation must not race those steps in clean parallel
builds.
++ zstd
++ snappy
++ lz4
+ arrow
+ tbb
+ glog
diff --git a/thirdparty/test/adbc-jni-config-test.sh
b/thirdparty/test/adbc-jni-config-test.sh
new file mode 100755
index 00000000000..6f2cf074e3c
--- /dev/null
+++ b/thirdparty/test/adbc-jni-config-test.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd)"
+EXTERNAL_FE_CONF="${ROOT}/regression-test/pipeline/external/conf/fe.conf"
+# shellcheck disable=SC2016 # Match the literal option before fe.conf expands
DORIS_HOME.
+JNI_LIBRARY_OPTION='-Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib'
+
+# External Regression replaces the packaged FE configuration instead of
merging it.
+# Keep the JNI override in that replacement so ADBC cannot fall back to the
jar binary.
+java_opts_assignment="$(grep '^JAVA_OPTS_FOR_JDK_17=' "${EXTERNAL_FE_CONF}" ||
true)"
+if [[ " ${java_opts_assignment} " != *" ${JNI_LIBRARY_OPTION} "* ]]; then
+ echo "FAIL: ${EXTERNAL_FE_CONF} drops the packaged ADBC JNI library path."
>&2
+ echo "Add ${JNI_LIBRARY_OPTION} to its FE JVM options." >&2
+ exit 1
+fi
+
+echo "PASS"
diff --git a/thirdparty/test/paimon-codec-dependency-test.sh
b/thirdparty/test/paimon-codec-dependency-test.sh
new file mode 100755
index 00000000000..4d3b92458b8
--- /dev/null
+++ b/thirdparty/test/paimon-codec-dependency-test.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)"
+
+fail() {
+ echo "FAIL: $*" >&2
+ exit 1
+}
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "${tmpdir}"' EXIT
+
+fixture="${tmpdir}/paimon-codec-dependency"
+focused_patch="${fixture}/codec-dependency.patch"
+mkdir -p "${fixture}/src/paimon"
+{
+ printf '%s\n' 'add_paimon_lib(paimon'
+ printf '%s\n' ' SOURCES'
+ printf '%s\n' ' ${PAIMON_COMMON_SRCS}'
+ printf '%s\n' ' ${PAIMON_CORE_SRCS}'
+ printf '%s\n' ' DEPENDENCIES'
+ printf '%s\n' ' arrow'
+ printf '%s\n' ' tbb'
+ printf '%s\n' ' glog'
+ printf '%s\n' ' fmt'
+ printf '%s\n' ' roaring_bitmap'
+ printf '%s\n' ' xxhash'
+ printf '%s\n' ' Threads::Threads'
+ printf '%s\n' ' RapidJSON'
+ printf '%s\n' ' STATIC_LINK_LIBS'
+ printf '%s\n' ' arrow)'
+} >"${fixture}/src/paimon/CMakeLists.txt"
+
+awk '
+ /^diff --git / {
+ if (found) {
+ exit
+ }
+ if ($0 == "diff --git a/src/paimon/CMakeLists.txt
b/src/paimon/CMakeLists.txt") {
+ found = 1
+ }
+ }
+ found { print }
+' "${ROOT}/patches/paimon-cpp-buildutils-static-deps.patch" >"${focused_patch}"
+[[ -s "${focused_patch}" ]] || fail "the Paimon codec dependency patch is
missing"
+(
+ cd "${fixture}"
+ patch -s -p1 <"${focused_patch}"
+) || fail "the Paimon codec dependency patch did not apply"
+
+dependencies="$(sed -n
'/^[[:space:]]*DEPENDENCIES$/,/^[[:space:]]*STATIC_LINK_LIBS$/p' \
+ "${fixture}/src/paimon/CMakeLists.txt")"
+grep -Eq '^[[:space:]]+zstd$' <<<"${dependencies}" ||
+ fail "paimon_objlib does not wait for the ZSTD headers"
+grep -Eq '^[[:space:]]+snappy$' <<<"${dependencies}" ||
+ fail "paimon_objlib does not wait for the Snappy headers"
+grep -Eq '^[[:space:]]+lz4$' <<<"${dependencies}" ||
+ fail "paimon_objlib does not wait for the LZ4 headers"
+
+echo "PASS"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
