pzhdfy commented on code in PR #66227:
URL: https://github.com/apache/doris/pull/66227#discussion_r4060187520
##########
thirdparty/build-thirdparty.sh:
##########
@@ -2169,6 +2169,222 @@ build_lance_c() {
fi
}
+# paimon-rust
+build_paimon_rust() {
+ check_if_source_exist "${PAIMON_RUST_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${PAIMON_RUST_SOURCE}"
+
+ rm -rf "${BUILD_DIR}"
+ mkdir -p "${BUILD_DIR}"
+
+ local cargo_bin="${PAIMON_RUST_CARGO:-${CARGO:-cargo}}"
+ if ! command -v "${cargo_bin}" >/dev/null 2>&1; then
+ echo "cargo is required to build paimon-rust. Install Rust 1.91.0 or
set PAIMON_RUST_CARGO."
+ exit 1
+ fi
+
+ local required_rust_version="1.91.0"
+ local cargo_env=(
+ "CARGO_BUILD_JOBS=${PARALLEL}"
+ "CARGO_TARGET_DIR=${PWD}/${BUILD_DIR}"
+ )
+ if command -v rustup >/dev/null 2>&1 && [[ -z "${RUSTUP_TOOLCHAIN}" ]];
then
+ if ! rustup toolchain list | grep -Eq '^1\.91\.0([[:space:]-]|$)'; then
+ rustup toolchain install "${required_rust_version}" --profile
minimal
+ fi
+ cargo_env+=("RUSTUP_TOOLCHAIN=${required_rust_version}")
+ fi
+
+ local cargo_version
+ if ! cargo_version="$(env "${cargo_env[@]}" "${cargo_bin}" --version | awk
'{print $2}')"; then
+ echo "failed to get cargo version for paimon-rust. Install Rust
${required_rust_version} or set PAIMON_RUST_CARGO/RUSTUP_TOOLCHAIN."
+ exit 1
+ fi
+ # Rust 1.91.0 is the minimum supported version. Allow newer toolchains when
+ # callers explicitly select one or rustup is unavailable on the system.
+ # NOTE: paimon_c and lance_c are both Rust staticlibs linked into the same
+ # BE binary; they must be built with the SAME rustc toolchain so the linker
+ # resolves both crates' std references against a single std copy. Mixing
+ # toolchains makes the precompiled std hashes differ and the linker pulls
+ # both std copies in, colliding on the unmangled `rust_eh_personality`
+ # (duplicate symbol). Build lance_c and paimon_rust with one toolchain.
+ if ! awk -v required="${required_rust_version}" -v
actual="${cargo_version}" 'BEGIN {
+ split(required, r, ".");
+ split(actual, a, ".");
+ for (i = 1; i <= 3; i++) {
+ if ((a[i] + 0) > (r[i] + 0)) {
+ exit 0;
+ }
+ if ((a[i] + 0) < (r[i] + 0)) {
+ exit 1;
+ }
+ }
+ exit 0;
+ }'; then
+ echo "paimon-rust requires Rust/Cargo ${required_rust_version} or
newer, but found ${cargo_version}."
+ echo "Install Rust ${required_rust_version} or set
PAIMON_RUST_CARGO/RUSTUP_TOOLCHAIN."
+ exit 1
+ fi
+
+ if [[ "${KERNEL}" != 'Darwin' ]]; then
+ cargo_env+=("CFLAGS=${CFLAGS:-} -std=gnu17")
+ fi
+
+ # paimon-vindex-core 0.4.0 uses the unstable `stdarch_neon_f16` intrinsics
+ # (vcvt_f32_f16 / vreinterpret_f16_u16) in its aarch64 NEON fast path,
which
+ # do not compile on stable Rust. On aarch64/arm64, replace the registry
+ # crate with a patched path source so the build succeeds. The patch swaps
+ # the unstable f16->f32 NEON convert for a stable scalar conversion; the
+ # rest of the NEON accumulation is left untouched. x86_64 and other arches
+ # are unaffected and keep using the pristine registry crate.
+ if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then
+ local vindex_override="${PWD}/.doris-vindex-override"
+ local vindex_crate
+ vindex_crate="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/cache" \
+ -type f -name 'paimon-vindex-core-0.4.0.crate' 2>/dev/null | head
-n1)"
+ if [[ -z "${vindex_crate}" ]]; then
+ # Ensure the .crate is downloaded into the registry cache first.
+ local fetch_args=(fetch)
+ if [[ "$(echo "${PAIMON_RUST_CARGO_OFFLINE}" | tr '[:lower:]'
'[:upper:]')" == "ON" ]]; then
+ fetch_args+=(--offline)
+ fi
+ env "${cargo_env[@]}" "${cargo_bin}" "${fetch_args[@]}"
+ vindex_crate="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/cache" \
+ -type f -name 'paimon-vindex-core-0.4.0.crate' 2>/dev/null |
head -n1)"
+ fi
+ if [[ -z "${vindex_crate}" ]]; then
+ echo "failed to locate paimon-vindex-core-0.4.0.crate in the cargo
registry cache"
+ exit 1
+ fi
+ # Verify the cached crate against the workspace Cargo.lock checksum
+ # before extraction. The cache lookup picks an ambient file by name,
+ # and once the [patch.crates-io] path override is applied, cargo
+ # build --locked no longer authenticates those bytes — without this
+ # check a stale or poisoned same-named cache (e.g. from another
+ # registry mirror) would enter libpaimon_c.a, and identical Doris
+ # sources could produce different artifacts. Iterate the candidates
+ # (multiple registries may cache the crate) and use the one whose
+ # sha256 matches the lock; fail when none does.
+ local vindex_checksum
+ vindex_checksum="$(awk '
+ $0 == "[[package]]" { in_pkg = 1; name = ""; version = "";
checksum = ""; next }
+ in_pkg && $1 == "name" { gsub(/[",]/, "", $3); name = $3 }
+ in_pkg && $1 == "version" { gsub(/[",]/, "", $3); version = $3 }
+ in_pkg && $1 == "checksum" { gsub(/[",]/, "", $3); checksum = $3 }
+ in_pkg && $0 == "" {
+ if (name == "paimon-vindex-core" && version == "0.4.0" &&
checksum != "") { print checksum; found = 1; exit }
+ in_pkg = 0
+ }
+ END {
+ if (!found && in_pkg && name == "paimon-vindex-core" &&
version == "0.4.0" && checksum != "") { print checksum }
+ }
+ ' Cargo.lock)"
+ if [[ -z "${vindex_checksum}" ]]; then
+ echo "failed to read the paimon-vindex-core 0.4.0 checksum from
Cargo.lock"
+ exit 1
+ fi
+ local vindex_verified=""
+ local candidate
+ while IFS= read -r candidate; do
+ local candidate_sum
+ if command -v sha256sum >/dev/null 2>&1; then
+ candidate_sum="$(sha256sum "${candidate}" | awk '{print $1}')"
+ else
+ candidate_sum="$(shasum -a 256 "${candidate}" | awk '{print
$1}')"
+ fi
+ if [[ "${candidate_sum}" == "${vindex_checksum}" ]]; then
+ vindex_verified="${candidate}"
+ break
+ fi
+ done < <(find "${CARGO_HOME:-$HOME/.cargo}/registry/cache" \
+ -type f -name 'paimon-vindex-core-0.4.0.crate' 2>/dev/null)
+ if [[ -z "${vindex_verified}" ]]; then
+ echo "no paimon-vindex-core-0.4.0.crate in the cargo registry
cache matches"
+ echo "the Cargo.lock checksum ${vindex_checksum}; refusing to
build from"
+ echo "unverified bytes (the aarch64 patch overrides the crate with
a path"
+ echo "dependency, which cargo build --locked cannot authenticate)"
+ exit 1
+ fi
+ vindex_crate="${vindex_verified}"
+ rm -rf "${vindex_override}"
+ mkdir -p "${vindex_override}"
+ tar xzf "${vindex_crate}" -C "${vindex_override}" --strip-components=1
+ (cd "${vindex_override}" && \
+ patch -p1 -s
<"${TP_PATCH_DIR}/paimon-vindex-core-0.4.0-aarch64-stable.patch")
+ # Inject the [patch.crates-io] override into the workspace manifest.
+ # Idempotent: skip if a previous run already injected it.
+ if ! grep -q "DORIS_PATCHED_VINDEX" Cargo.toml; then
+ cat >>Cargo.toml <<'EOF'
+
+# DORIS_PATCHED_VINDEX: override the registry crate with a build that compiles
+# on stable Rust for aarch64 (avoids the unstable stdarch_neon_f16 intrinsics).
+[patch.crates-io]
+paimon-vindex-core = { path = ".doris-vindex-override" }
+EOF
+ fi
+ # Record the path source in Cargo.lock so --locked stays satisfied.
+ env "${cargo_env[@]}" "${cargo_bin}" update \
Review Comment:
Fixed — canUseRust gates fallback reads to JNI through both shapes: a
wrapped split (split instanceof FallbackReadFileStoreTable.FallbackSplit, the
public interface every FallbackDataSplit implements) and the table wrapper
itself (processedTable instanceof FallbackReadFileStoreTable — both read sides
of a fallback table produce wrapped splits, so the whole wrapper routes to JNI
until the rust ABI can represent both sides). The routing test exercises the
genuine wire shape rather than a mock: an ordinary DataSplit serialized, the
isFallback byte appended exactly like FallbackDataSplit.serialize, and the
result deserialized through the public FallbackDataSplit.deserialize factory —
asserting JNI routing for both the split and the table-wrapper shapes;
PaimonScanNodeTest 51/51.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
