This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new e1f727cbb4 GH-47588: [C++] Bump mimalloc version to 3.1.5 (#47589)
e1f727cbb4 is described below
commit e1f727cbb447d2385949a54d8f4be2fdc6cefe29
Author: Antoine Pitrou <[email protected]>
AuthorDate: Thu Oct 9 11:47:04 2025 +0200
GH-47588: [C++] Bump mimalloc version to 3.1.5 (#47589)
### Rationale for this change
According to https://github.com/microsoft/mimalloc/issues/1073 , mimalloc
v3 is preferred over v2 for production usage.
There are reports of higher than expected memory consumption with mimalloc
2.2.x, notably when reading Parquet data (example: GH-47266).
### What changes are included in this PR?
Bump to mimalloc 3.1.5, which is the latest mimalloc 3.1.x release as of
this writing.
### Are these changes tested?
Yes, by existing tests and CI.
### Are there any user-facing changes?
Hopefully not, besides a potential reduction in memory usage due to
improvements in mimalloc v3.
* GitHub Issue: #47588
Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
ci/conda_env_cpp.txt | 1 +
ci/docker/alpine-linux-3.22-cpp.dockerfile | 1 +
ci/docker/debian-12-cpp.dockerfile | 1 +
ci/docker/debian-13-cpp.dockerfile | 1 +
ci/docker/debian-experimental-cpp.dockerfile | 1 +
ci/docker/fedora-42-cpp.dockerfile | 1 +
ci/docker/ubuntu-22.04-cpp-minimal.dockerfile | 1 +
ci/docker/ubuntu-22.04-cpp.dockerfile | 1 +
ci/docker/ubuntu-24.04-cpp-minimal.dockerfile | 1 +
ci/docker/ubuntu-24.04-cpp.dockerfile | 1 +
ci/scripts/msys2_setup.sh | 1 +
cpp/cmake_modules/ThirdpartyToolchain.cmake | 13 +++++-
.../cmake_modules/mimalloc-1138.patch | 51 +++++++---------------
cpp/thirdparty/versions.txt | 4 +-
14 files changed, 40 insertions(+), 39 deletions(-)
diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt
index 6e23e920a4..52b4cdbba8 100644
--- a/ci/conda_env_cpp.txt
+++ b/ci/conda_env_cpp.txt
@@ -41,6 +41,7 @@ meson
ninja
nodejs
orc<2.1.0
+patch
pkg-config
python
rapidjson
diff --git a/ci/docker/alpine-linux-3.22-cpp.dockerfile
b/ci/docker/alpine-linux-3.22-cpp.dockerfile
index f03e1913e2..48907e61a4 100644
--- a/ci/docker/alpine-linux-3.22-cpp.dockerfile
+++ b/ci/docker/alpine-linux-3.22-cpp.dockerfile
@@ -48,6 +48,7 @@ RUN apk add \
musl-locales \
nlohmann-json \
openssl-dev \
+ patch \
perl \
pkgconfig \
protobuf-dev \
diff --git a/ci/docker/debian-12-cpp.dockerfile
b/ci/docker/debian-12-cpp.dockerfile
index 15716151fc..44c845bb17 100644
--- a/ci/docker/debian-12-cpp.dockerfile
+++ b/ci/docker/debian-12-cpp.dockerfile
@@ -85,6 +85,7 @@ RUN apt-get update -y -q && \
ninja-build \
nlohmann-json3-dev \
npm \
+ patch \
pkg-config \
protobuf-compiler-grpc \
python3-dev \
diff --git a/ci/docker/debian-13-cpp.dockerfile
b/ci/docker/debian-13-cpp.dockerfile
index 3e5c645c81..ca96b4177f 100644
--- a/ci/docker/debian-13-cpp.dockerfile
+++ b/ci/docker/debian-13-cpp.dockerfile
@@ -87,6 +87,7 @@ RUN apt-get update -y -q && \
nlohmann-json3-dev \
npm \
opentelemetry-cpp-dev \
+ patch \
pkg-config \
protobuf-compiler-grpc \
python3-dev \
diff --git a/ci/docker/debian-experimental-cpp.dockerfile
b/ci/docker/debian-experimental-cpp.dockerfile
index 2721b1d5f2..743f5ddd3b 100644
--- a/ci/docker/debian-experimental-cpp.dockerfile
+++ b/ci/docker/debian-experimental-cpp.dockerfile
@@ -79,6 +79,7 @@ RUN if [ -n "${gcc}" ]; then \
nlohmann-json3-dev \
npm \
opentelemetry-cpp-dev \
+ patch \
pkg-config \
protobuf-compiler-grpc \
python3-dev \
diff --git a/ci/docker/fedora-42-cpp.dockerfile
b/ci/docker/fedora-42-cpp.dockerfile
index 82e3fa9f7a..cabb066fec 100644
--- a/ci/docker/fedora-42-cpp.dockerfile
+++ b/ci/docker/fedora-42-cpp.dockerfile
@@ -53,6 +53,7 @@ RUN dnf update -y && \
make \
ninja-build \
openssl-devel \
+ patch \
protobuf-devel \
python \
python-devel \
diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
index 2a90a5637d..d38dd418e2 100644
--- a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
+++ b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
@@ -33,6 +33,7 @@ RUN apt-get update -y -q && \
git \
libssl-dev \
libcurl4-openssl-dev \
+ patch \
python3-pip \
python3-venv \
tzdata \
diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile
b/ci/docker/ubuntu-22.04-cpp.dockerfile
index 44c1a452c1..88a27efe33 100644
--- a/ci/docker/ubuntu-22.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -107,6 +107,7 @@ RUN apt-get update -y -q && \
ninja-build \
nlohmann-json3-dev \
npm \
+ patch \
pkg-config \
protobuf-compiler \
protobuf-compiler-grpc \
diff --git a/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile
b/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile
index a1fd178a2c..5e114d5dcd 100644
--- a/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile
+++ b/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile
@@ -33,6 +33,7 @@ RUN apt-get update -y -q && \
git \
libssl-dev \
libcurl4-openssl-dev \
+ patch \
python3-pip \
python3-venv \
tzdata \
diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile
b/ci/docker/ubuntu-24.04-cpp.dockerfile
index 6bc49a4c84..0347d452d7 100644
--- a/ci/docker/ubuntu-24.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-24.04-cpp.dockerfile
@@ -108,6 +108,7 @@ RUN apt-get update -y -q && \
ninja-build \
nlohmann-json3-dev \
npm \
+ patch \
pkg-config \
protobuf-compiler \
protobuf-compiler-grpc \
diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh
index 0ce9343a7f..b4634070a8 100755
--- a/ci/scripts/msys2_setup.sh
+++ b/ci/scripts/msys2_setup.sh
@@ -51,6 +51,7 @@ case "${target}" in
packages+=("${MINGW_PACKAGE_PREFIX}-xsimd")
packages+=("${MINGW_PACKAGE_PREFIX}-uriparser")
packages+=("${MINGW_PACKAGE_PREFIX}-zstd")
+ packages+=("patch")
if [ "${target}" != "ruby" ]; then
# We don't update the exiting packages for Ruby because
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 7d110e8dbb..4ced2a66bf 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -2276,9 +2276,9 @@ if(ARROW_MIMALLOC)
endif()
set(MIMALLOC_PREFIX
"${CMAKE_CURRENT_BINARY_DIR}/mimalloc_ep/src/mimalloc_ep")
- set(MIMALLOC_INCLUDE_DIR "${MIMALLOC_PREFIX}/include/mimalloc-2.2")
+ set(MIMALLOC_INCLUDE_DIR "${MIMALLOC_PREFIX}/include")
set(MIMALLOC_STATIC_LIB
-
"${MIMALLOC_PREFIX}/lib/mimalloc-2.2/${CMAKE_STATIC_LIBRARY_PREFIX}${MIMALLOC_LIB_BASE_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
+
"${MIMALLOC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${MIMALLOC_LIB_BASE_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(MIMALLOC_C_FLAGS ${EP_C_FLAGS})
@@ -2287,10 +2287,18 @@ if(ARROW_MIMALLOC)
set(MIMALLOC_C_FLAGS "${MIMALLOC_C_FLAGS} -DERROR_COMMITMENT_MINIMUM=635")
endif()
+ set(MIMALLOC_PATCH_COMMAND "")
+ if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+ find_program(PATCH patch REQUIRED)
+ set(MIMALLOC_PATCH_COMMAND ${PATCH} -p1 -i
+ ${CMAKE_CURRENT_LIST_DIR}/mimalloc-1138.patch)
+ endif()
+
set(MIMALLOC_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_C_FLAGS=${MIMALLOC_C_FLAGS}"
"-DCMAKE_INSTALL_PREFIX=${MIMALLOC_PREFIX}"
+ -DMI_INSTALL_TOPLEVEL=ON
-DMI_OVERRIDE=OFF
-DMI_LOCAL_DYNAMIC_TLS=ON
-DMI_BUILD_OBJECT=OFF
@@ -2303,6 +2311,7 @@ if(ARROW_MIMALLOC)
${EP_COMMON_OPTIONS}
URL ${MIMALLOC_SOURCE_URL}
URL_HASH "SHA256=${ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM}"
+ PATCH_COMMAND ${MIMALLOC_PATCH_COMMAND}
CMAKE_ARGS ${MIMALLOC_CMAKE_ARGS}
BUILD_BYPRODUCTS "${MIMALLOC_STATIC_LIB}")
diff --git a/ci/conda_env_cpp.txt b/cpp/cmake_modules/mimalloc-1138.patch
similarity index 55%
copy from ci/conda_env_cpp.txt
copy to cpp/cmake_modules/mimalloc-1138.patch
index 6e23e920a4..1ffa4bffbb 100644
--- a/ci/conda_env_cpp.txt
+++ b/cpp/cmake_modules/mimalloc-1138.patch
@@ -15,38 +15,19 @@
# specific language governing permissions and limitations
# under the License.
-aws-sdk-cpp=1.11.68
-azure-core-cpp>=1.10.3
-azure-identity-cpp>=1.6.0
-azure-storage-blobs-cpp>=12.10.0
-azure-storage-common-cpp>=12.5.0
-azure-storage-files-datalake-cpp>=12.9.0
-benchmark>=1.6.0,!=1.8.4
-boost-cpp>=1.68.0
-brotli
-bzip2
-c-ares
-cmake
-gflags
-glog
-gmock>=1.10.0
-google-cloud-cpp>=1.34.0
-gtest>=1.10.0
-libgrpc
-libprotobuf
-libutf8proc
-lz4-c
-make
-meson
-ninja
-nodejs
-orc<2.1.0
-pkg-config
-python
-rapidjson
-re2
-snappy
-thrift-cpp>=0.11.0
-xsimd
-zlib
-zstd
+Fix for https://github.com/microsoft/mimalloc/issues/1138
+
+diff --git a/src/arena.c b/src/arena.c
+index b26f4442..d7e99b55 100644
+--- a/src/arena.c
++++ b/src/arena.c
+@@ -797,6 +797,9 @@ mi_page_t* _mi_arenas_page_alloc(mi_heap_t* heap, size_t
block_size, size_t bloc
+ else {
+ page = mi_arenas_page_singleton_alloc(heap, block_size, block_alignment);
+ }
++ if mi_unlikely(page == NULL) {
++ return NULL;
++ }
+ // mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc ==
tld->subproc);
+ mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
+ mi_assert_internal(_mi_ptr_page(page)==page);
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
index 092f7939b4..7ba1f4f876 100644
--- a/cpp/thirdparty/versions.txt
+++ b/cpp/thirdparty/versions.txt
@@ -82,8 +82,8 @@ ARROW_JEMALLOC_BUILD_VERSION=5.3.0
ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa
ARROW_LZ4_BUILD_VERSION=v1.10.0
ARROW_LZ4_BUILD_SHA256_CHECKSUM=537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b
-ARROW_MIMALLOC_BUILD_VERSION=v2.2.4
-ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=754a98de5e2912fddbeaf24830f982b4540992f1bab4a0a8796ee118e0752bda
+ARROW_MIMALLOC_BUILD_VERSION=v3.1.5
+ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=1c6949032069d5ebea438ec5cedd602d06f40a92ddf0f0d9dcff0993e5f6635c
ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.12.0
ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=4b92eb0c06d10683f7447ce9406cb97cd4b453be18d7279320f7b2f025c10187
ARROW_OPENTELEMETRY_BUILD_VERSION=v1.21.0
