commit: c283279b54747119786340da3167053f002fb829 Author: Patrick Lauer <patrick <AT> gentoo <DOT> org> AuthorDate: Sat May 3 12:27:54 2025 +0000 Commit: Patrick Lauer <patrick <AT> gentoo <DOT> org> CommitDate: Sat May 3 12:28:55 2025 +0000 URL: https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=c283279b
sys-cluster/ceph: Add python-3.13 support Series of patches picked out of ceph git repo. Closes: https://bugs.gentoo.org/952730 Signed-off-by: Patrick Lauer <patrick <AT> gentoo.org> sys-cluster/ceph/ceph-19.2.2-r1.ebuild | 526 +++++++++++++++++++++++ sys-cluster/ceph/files/ceph-19.2.2-py313-1.patch | 95 ++++ sys-cluster/ceph/files/ceph-19.2.2-py313-2.patch | 61 +++ sys-cluster/ceph/files/ceph-19.2.2-py313-3.patch | 252 +++++++++++ 4 files changed, 934 insertions(+) diff --git a/sys-cluster/ceph/ceph-19.2.2-r1.ebuild b/sys-cluster/ceph/ceph-19.2.2-r1.ebuild new file mode 100644 index 000000000000..469409d5ddff --- /dev/null +++ b/sys-cluster/ceph/ceph-19.2.2-r1.ebuild @@ -0,0 +1,526 @@ +# Copyright 1999-2025 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +PYTHON_COMPAT=( python3_{10..13} ) +LUA_COMPAT=( lua5-{3..4} ) + +inherit check-reqs bash-completion-r1 cmake flag-o-matic lua-single \ + python-r1 udev readme.gentoo-r1 toolchain-funcs systemd tmpfiles + +XSIMD_HASH="aeec9c872c8b475dedd7781336710f2dd2666cb2" + +DESCRIPTION="Ceph distributed filesystem" +HOMEPAGE="https://ceph.com/"; + +SRC_URI=" + https://download.ceph.com/tarballs/${P}.tar.gz + parquet? ( https://github.com/xtensor-stack/xsimd/archive/${XSIMD_HASH}.tar.gz -> ceph-xsimd-${PV}.tar.gz + mirror://apache/arrow/arrow-17.0.0/apache-arrow-17.0.0.tar.gz ) +" + +LICENSE="Apache-2.0 LGPL-2.1 CC-BY-SA-3.0 GPL-2 GPL-2+ LGPL-2+ LGPL-2.1 LGPL-3 GPL-3 BSD Boost-1.0 MIT public-domain" +SLOT="0" +KEYWORDS="~amd64 ~arm64 ~ppc64" + +CPU_FLAGS_X86=(avx2 avx512f pclmul sse{,2,3,4_1,4_2} ssse3) + +IUSE=" + babeltrace +cephfs custom-cflags diskprediction dpdk fuse grafana + jemalloc jaeger kafka kerberos ldap lttng +mgr +parquet pmdk rabbitmq + +radosgw rbd-rwl rbd-ssd rdma rgw-lua selinux +ssl spdk +sqlite +system-boost + systemd +tcmalloc test +uring xfs zbd +" + +IUSE+="$(printf "cpu_flags_x86_%s\n" ${CPU_FLAGS_X86[@]})" + +DEPEND=" + ${LUA_DEPS} + ${PYTHON_DEPS} + acct-group/ceph + acct-user/ceph + virtual/libudev:= + app-arch/bzip2:= + app-arch/lz4:= + app-arch/snappy:= + >=app-arch/snappy-1.1.9-r1 + app-arch/zstd:= + app-shells/bash:0 + app-misc/jq:= + dev-cpp/gflags:= + dev-db/lmdb:= + dev-lang/jsonnet:= + dev-libs/libaio:= + dev-libs/libnl:3= + dev-libs/libxml2:= + dev-libs/libevent:= + dev-libs/libutf8proc:= + dev-libs/nss:= + dev-libs/openssl:= + <dev-libs/rocksdb-7.9.3:= + dev-libs/thrift:= + dev-libs/xmlsec:=[openssl] + dev-cpp/yaml-cpp:= + dev-python/natsort[${PYTHON_USEDEP}] + dev-python/pyyaml[${PYTHON_USEDEP}] + dev-vcs/git + net-dns/c-ares:= + net-libs/gnutls:= + sys-auth/oath-toolkit:= + sys-apps/coreutils + sys-apps/hwloc:= + sys-apps/keyutils:= + sys-apps/util-linux:= + sys-libs/libcap-ng:= + sys-libs/ncurses:0= + sys-libs/zlib:= + sys-process/numactl:= + virtual/libcrypt:= + x11-libs/libpciaccess:= + babeltrace? ( dev-util/babeltrace:0/1 ) + fuse? ( sys-fs/fuse:3= ) + jemalloc? ( dev-libs/jemalloc:= ) + !jemalloc? ( >=dev-util/google-perftools-2.6.1:= ) + jaeger? ( + dev-cpp/nlohmann_json:= + <dev-cpp/opentelemetry-cpp-1.10.0:=[jaeger] + ) + kafka? ( dev-libs/librdkafka:= ) + kerberos? ( virtual/krb5 ) + ldap? ( net-nds/openldap:= ) + lttng? ( dev-util/lttng-ust:= ) + parquet? ( + >=app-arch/lz4-1.10 + dev-cpp/xsimd + dev-libs/re2:= + ) + pmdk? ( + >=dev-libs/pmdk-1.10.0:= + sys-block/ndctl:= + ) + rabbitmq? ( net-libs/rabbitmq-c:= ) + radosgw? ( + dev-libs/icu:= + dev-libs/expat:= + net-misc/curl:=[curl_ssl_openssl] + ) + rbd-rwl? ( dev-libs/pmdk:= ) + rdma? ( sys-cluster/rdma-core:= ) + spdk? ( dev-util/cunit ) + sqlite? ( dev-db/sqlite:= ) + system-boost? ( dev-libs/boost:=[context,python,${PYTHON_USEDEP},zlib] ) + uring? ( sys-libs/liburing:= ) + xfs? ( sys-fs/xfsprogs:= ) + zbd? ( sys-block/libzbd:= ) +" +BDEPEND=" + amd64? ( dev-lang/nasm ) + x86? ( dev-lang/yasm ) + app-alternatives/cpio + dev-debug/valgrind + >=dev-build/cmake-3.5.0 + dev-python/cython[${PYTHON_USEDEP}] + dev-python/setuptools[${PYTHON_USEDEP}] + dev-python/sphinx + dev-util/gperf + dev-util/ragel + sys-apps/coreutils + sys-apps/grep + sys-apps/util-linux + sys-apps/which + app-alternatives/bc + sys-devel/patch + virtual/pkgconfig + jaeger? ( + app-alternatives/yacc + app-alternatives/lex + ) + test? ( + dev-util/cunit + dev-python/coverage[${PYTHON_USEDEP}] + dev-python/virtualenv[${PYTHON_USEDEP}] + dev-python/requests-mock[${PYTHON_USEDEP}] + sys-apps/grep[pcre] + sys-fs/btrfs-progs + ) +" +RDEPEND=" + ${DEPEND} + app-admin/sudo + net-misc/socat + sys-apps/gptfdisk + sys-apps/nvme-cli + >=sys-apps/smartmontools-7.0 + sys-block/parted + sys-fs/cryptsetup + sys-fs/lsscsi + sys-fs/lvm2[lvm] + app-alternatives/awk + dev-python/bcrypt[${PYTHON_USEDEP}] + dev-python/cherrypy[${PYTHON_USEDEP}] + dev-python/python-dateutil[${PYTHON_USEDEP}] + dev-python/flask[${PYTHON_USEDEP}] + dev-python/jinja2[${PYTHON_USEDEP}] + dev-python/pecan[${PYTHON_USEDEP}] + dev-python/prettytable[${PYTHON_USEDEP}] + dev-python/pyopenssl[${PYTHON_USEDEP}] + dev-python/requests[${PYTHON_USEDEP}] + dev-python/werkzeug[${PYTHON_USEDEP}] + mgr? ( + dev-python/jsonpatch[${PYTHON_USEDEP}] + dev-python/more-itertools[${PYTHON_USEDEP}] + dev-python/numpy[${PYTHON_USEDEP}] + dev-python/pyjwt[${PYTHON_USEDEP}] + dev-python/routes[${PYTHON_USEDEP}] + diskprediction? ( + >=dev-python/scipy-1.4.0[${PYTHON_USEDEP}] + ) + dev-python/scikit-learn[${PYTHON_USEDEP}] + dev-python/six[${PYTHON_USEDEP}] + ) + selinux? ( sec-policy/selinux-ceph ) +" +REQUIRED_USE=" + ${PYTHON_REQUIRED_USE} + ${LUA_REQUIRED_USE} + ?? ( jemalloc tcmalloc ) + diskprediction? ( mgr ) + kafka? ( radosgw ) + mgr? ( cephfs ) + rabbitmq? ( radosgw ) + rgw-lua? ( radosgw ) +" + +RESTRICT=" + !test? ( test ) +" + +# tests need root access, and network access +RESTRICT+="test" + +# false positives unless all USE flags are on +CMAKE_WARN_UNUSED_CLI=no + +PATCHES=( + "${FILESDIR}/ceph-12.2.0-use-provided-cpu-flag-values.patch" + "${FILESDIR}/ceph-14.2.0-cflags.patch" + "${FILESDIR}/ceph-17.2.1-no-virtualenvs.patch" + "${FILESDIR}/ceph-13.2.2-dont-install-sysvinit-script.patch" + "${FILESDIR}/ceph-14.2.0-dpdk-cflags.patch" + "${FILESDIR}/ceph-16.2.0-rocksdb-cmake.patch" + "${FILESDIR}/ceph-16.2.0-spdk-tinfo.patch" + "${FILESDIR}/ceph-16.2.0-jaeger-system-boost.patch" + "${FILESDIR}/ceph-17.2.0-pybind-boost-1.74.patch" + "${FILESDIR}/ceph-17.2.0-findre2.patch" + "${FILESDIR}/ceph-18.2.0-system-opentelemetry.patch" + "${FILESDIR}/ceph-17.2.0-osd_class_dir.patch" + "${FILESDIR}/ceph-17.2.0-gcc12-header.patch" + "${FILESDIR}/ceph-17.2.3-flags.patch" + # https://bugs.gentoo.org/866165 + "${FILESDIR}/ceph-17.2.5-suppress-cmake-warning.patch" + "${FILESDIR}/ceph-17.2.5-gcc13-deux.patch" + # https://bugs.gentoo.org/905626 + "${FILESDIR}/ceph-17.2.6-arrow-flatbuffers-c++14.patch" + # https://bugs.gentoo.org/868891 + "${FILESDIR}/ceph-17.2.6-cmake.patch" + "${FILESDIR}/ceph-18.2.0-cyclic-deps.patch" + # https://bugs.gentoo.org/907739 + "${FILESDIR}/ceph-18.2.0-cython3.patch" + # https://bugs.gentoo.org/936889 + "${FILESDIR}/ceph-18.2.4-liburing.patch" + "${FILESDIR}/ceph-18.2.4-spdk.patch" + # https://bugs.gentoo.org/941069 + "${FILESDIR}/ceph-19.2.0-importlib.patch" + "${FILESDIR}/ceph-19.2.1-uuid.patch" + "${FILESDIR}/ceph-19.2.1-graylog.patch" + "${FILESDIR}/ceph-19.2.1-librbd.patch" + "${FILESDIR}/ceph-19.2.1-rgw.patch" + "${FILESDIR}/ceph-19.2.1-immutableobjectcache.patch" + "${FILESDIR}/ceph-19.2.1-mgr.patch" + "${FILESDIR}/ceph-19.2.1-exporter.patch" + "${FILESDIR}/ceph-19.2.1-isa-l.patch" + "${FILESDIR}/ceph-19.2.1-quiet-stringop-truncation.patch" + "${FILESDIR}/ceph-19.2.2-py313-1.patch" + "${FILESDIR}/ceph-19.2.2-py313-2.patch" + "${FILESDIR}/ceph-19.2.2-py313-3.patch" +) + +check-reqs_export_vars() { + CHECKREQS_DISK_BUILD="6G" + + if use system-boost; then + CHECKREQS_DISK_USR="350M" + else + CHECKREQS_DISK_USR="510M" + fi + + export CHECKREQS_DISK_BUILD CHECKREQS_DISK_USR +} + +pkg_pretend() { + check-reqs_export_vars + check-reqs_pkg_pretend +} + +pkg_setup() { + python_setup + lua_setup + check-reqs_export_vars + check-reqs_pkg_setup +} + +src_prepare() { + cmake_src_prepare + + if use system-boost; then + find "${S}" -name '*.cmake' -or -name 'CMakeLists.txt' -print0 \ + | xargs --null sed -r \ + -e 's|Boost::|boost_|g' \ + -e 's|Boost_|boost_|g' \ + -e 's|[Bb]oost_boost|boost_system|g' \ + -i || die + fi + + if ! use systemd; then + find "${S}"/src/ceph-volume/ceph_volume -name '*.py' -print0 \ + | xargs --null sed \ + -e '/^from ceph_volume.systemd import systemctl/ d' \ + -i || die + fi + + sed -r -e "s:DESTINATION .+\\):DESTINATION $(get_bashcompdir)\\):" \ + -i src/bash_completion/CMakeLists.txt || die + + sed -e "s:objdump -p:$(tc-getOBJDUMP) -p:" -i CMakeLists.txt || die + + # force lua version to use selected version + local lua_version + lua_version=$(ver_cut 1-2 $(lua_get_version)) + sed "s:find_package(Lua [0-9][.][0-9] REQUIRED):find_package(Lua ${lua_version} EXACT REQUIRED):" \ + -i src/CMakeLists.txt + + if use spdk; then + # https://bugs.gentoo.org/871942 + sed -i 's/[#]ifndef HAVE_ARC4RANDOM/#if 0/' src/spdk/lib/iscsi/iscsi.c || die + # unittests fail to build (??!?) + sed -i -e 's/CONFIG_UNIT_TESTS=y/CONFIG_UNIT_TESTS=n/' src/spdk/CONFIG || die + fi + + # remove tests that need root access + rm src/test/cli/ceph-authtool/cap*.t || die + + if use parquet; then + # hammer in newer version of parquet/arrow + rm -rf src/arrow/ + mv "${WORKDIR}/apache-arrow-17.0.0" src/arrow || die + fi + + # newer boost don't support no header-only + sed -i -e 's~#include <boost/url/src.hpp>~#include <boost/url.hpp>~' src/mds/BoostUrlImpl.cc || die + + # everyone forgot to link to boost_url + sed -i -e 's~target_link_libraries(ceph-mds mds ${CMAKE_DL_LIBS} global-static ceph-common~target_link_libraries(ceph-mds mds ${CMAKE_DL_LIBS} global-static ceph-common boost_url~' src/CMakeLists.txt || die + sed -i -e 's/target_link_libraries(journal cls_journal_client)/target_link_libraries(journal cls_journal_client boost_url)/' src/journal/CMakeLists.txt || die + sed -i -e 's/${BLKID_LIBRARIES} ${CMAKE_DL_LIBS})/${BLKID_LIBRARIES} ${CMAKE_DL_LIBS} boost_url)/g' src/tools/cephfs/CMakeLists.txt || die +} + +ceph_src_configure() { + local mycmakeargs=( + # Don't break installed bundled libraries (bug #942680) + -DBUILD_SHARED_LIBS=OFF + -DWITH_BABELTRACE:BOOL=$(usex babeltrace) + -DWITH_BLUESTORE_PMEM:BOOL=$(usex pmdk) + -DWITH_CEPHFS:BOOL=$(usex cephfs) + -DWITH_CEPHFS_SHELL:BOOL=$(usex cephfs) + -DWITH_DPDK:BOOL=$(usex dpdk) + -DWITH_SPDK:BOOL=$(usex spdk) + -DWITH_FUSE:BOOL=$(usex fuse) + -DWITH_LTTNG:BOOL=$(usex lttng) + -DWITH_GSSAPI:BOOL=$(usex kerberos) + -DWITH_GRAFANA:BOOL=$(usex grafana) + -DWITH_MGR:BOOL=$(usex mgr) + -DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF + -DWITH_OPENLDAP:BOOL=$(usex ldap) + -DWITH_PYTHON3:STRING=3 + -DWITH_RADOSGW:BOOL=$(usex radosgw) + -DWITH_RADOSGW_AMQP_ENDPOINT:BOOL=$(usex rabbitmq) + -DWITH_RADOSGW_KAFKA_ENDPOINT:BOOL=$(usex kafka) + -DWITH_RADOSGW_LUA_PACKAGES:BOOL=$(usex rgw-lua "$(usex radosgw)" "NO") + -DWITH_RBD_RWL:BOOL=$(usex rbd-rwl) + -DWITH_RBD_SSD_CACHE:BOOL=$(usex rbd-ssd) + -DWITH_SYSTEMD:BOOL=$(usex systemd) + -DWITH_TESTS:BOOL=$(usex test) + -DWITH_LIBURING:BOOL=$(usex uring) + -DWITH_SYSTEM_LIBURING:BOOL=$(usex uring) + -DWITH_LIBCEPHSQLITE:BOOL=$(usex sqlite) + -DWITH_XFS:BOOL=$(usex xfs) + -DWITH_ZBD:BOOL=$(usex zbd) + -DENABLE_SHARED:BOOL=ON + -DALLOCATOR:STRING=$(usex tcmalloc 'tcmalloc' "$(usex jemalloc 'jemalloc' 'libc')") + -DWITH_SYSTEM_PMDK:BOOL=$(usex pmdk 'YES' "$(usex rbd-rwl '')") + -DWITH_SYSTEM_BOOST:BOOL=$(usex system-boost) + -DWITH_SYSTEM_ROCKSDB:BOOL=ON + -DWITH_SYSTEM_ZSTD:BOOL=ON + -DWITH_RDMA:BOOL=$(usex rdma) + -DCMAKE_INSTALL_DOCDIR:PATH="${EPREFIX}/usr/share/doc/${PN}-${PVR}" + -DCMAKE_INSTALL_SYSCONFDIR:PATH="${EPREFIX}/etc" + # use the bundled libfmt for now since they seem to constantly break their API + -DCMAKE_DISABLE_FIND_PACKAGE_fmt=ON + -Wno-dev + -DCEPHADM_BUNDLED_DEPENDENCIES=none + ) + + # this breaks when re-configuring for python impl + if [[ ${EBUILD_PHASE} == configure ]]; then + mycmakeargs+=( + -DWITH_JAEGER:BOOL=$(usex jaeger) + -DWITH_RADOSGW_SELECT_PARQUET:BOOL=$(usex parquet) + ) + else + mycmakeargs+=( + -DWITH_RADOSGW_SELECT_PARQUET:BOOL=OFF + -DWITH_JAEGER:BOOL=OFF + # don't want to warn about unused CLI when reconfiguring for python + -DCMAKE_WARN_UNUSED_CLI:BOOL=OFF + ) + fi + + # conditionally used cmake args + use test && mycmakearts+=( -DWITH_SYSTEM_GTEST:BOOL=$(usex test) ) + use systemd && mycmakeargs+=( -DSYSTEMD_SYSTEM_UNIT_DIR:PATH=$(systemd_get_systemunitdir) ) + + if use amd64 || use x86; then + local flag + for flag in "${CPU_FLAGS_X86[@]}"; do + case "${flag}" in + avx*) + local var=${flag%f} + mycmakeargs+=( + "-DHAVE_NASM_X64_${var^^}:BOOL=$(usex cpu_flags_x86_${flag})" + ) + ;; + *) mycmakeargs+=( + "-DHAVE_INTEL_${flag^^}:BOOL=$(usex cpu_flags_x86_${flag})" + );; + esac + done + fi + + # needed for >=glibc-2.32 + has_version '>=sys-libs/glibc-2.32' && mycmakeargs+=( -DWITH_REENTRANT_STRSIGNAL:BOOL=ON ) + + rm -f "${BUILD_DIR:-${S}}/CMakeCache.txt" \ + || die "failed to remove cmake cache" + + # hopefully this will not be necessary in the next release + use parquet && export ARROW_XSIMD_URL="file:///${DISTDIR}/ceph-xsimd-${PV}.tar.gz" + + # https://bugs.gentoo.org/927066 + filter-lto + + cmake_src_configure + + # bug #630232 + sed -i "s:\"${T//:\\:}/${EPYTHON}/bin/python\":\"${PYTHON}\":" \ + "${BUILD_DIR:-${S}}"/include/acconfig.h \ + || die "sed failed" +} + +src_configure() { + use custom-cflags || strip-flags + ceph_src_configure +} + +src_compile() { + cmake_build all + + # we have to do this here to prevent from building everything multiple times + python_copy_sources + python_foreach_impl python_compile +} + +python_compile() { + local CMAKE_USE_DIR="${S}" + ceph_src_configure + + cmake_build src/pybind/CMakeFiles/cython_modules + cmake_build cephadm +} + +src_install() { + python_foreach_impl python_install + + python_setup + cmake_src_install + + python_optimize + + find "${ED}" -name '*.la' -type f -delete || die + + exeinto /usr/$(get_libdir)/ceph + newexe "${BUILD_DIR}/bin/init-ceph" init-ceph + + insinto /etc/logrotate.d/ + newins "${FILESDIR}"/ceph.logrotate-r2 ${PN} + + keepdir /var/lib/${PN}{,/tmp} /var/log/ceph/stat /var/log/ceph/console + + fowners -R ceph:ceph /var/log/ceph + + newinitd "${FILESDIR}/rbdmap.initd-r1" rbdmap + newinitd "${FILESDIR}/${PN}.initd-r14" ${PN} + newconfd "${FILESDIR}/${PN}.confd-r5" ${PN} + + insinto /etc/sudoers.d + doins sudoers.d/* + + insinto /etc/sysctl.d + newins "${FILESDIR}"/sysctld 90-${PN}.conf + + use tcmalloc && newenvd "${FILESDIR}"/envd-tcmalloc 99${PN}-tcmalloc + + # units aren't installed by the build system unless systemd is enabled + # so no point installing these with the USE flag disabled + if use systemd; then + systemd_install_serviced "${FILESDIR}/ceph-mds_at.service.conf" "[email protected]" + systemd_install_serviced "${FILESDIR}/ceph-osd_at.service.conf" "[email protected]" + fi + + udev_dorules udev/*.rules + newtmpfiles "${FILESDIR}"/ceph-tmpfilesd ${PN}.conf + + readme.gentoo_create_doc + + # bug #630232 + sed -i -r "s:${T//:/\\:}/${EPYTHON}:/usr:" "${ED}"/usr/bin/ceph{,-crash} \ + || die "sed failed" + + python_fix_shebang "${ED}"/usr/{,s}bin/ + + # python_fix_shebang apparently is not idempotent + local shebang_regex='(/usr/lib/python-exec/python[0-9]\.[0-9]/python)[0-9]\.[0-9]' + grep -r -E -l --null "${shebang_regex}" "${ED}"/usr/{s,}bin/ \ + | xargs --null --no-run-if-empty -- sed -i -r "s:${shebang_regex}:\1:" || die + + local -a rados_classes=( "${ED}/usr/$(get_libdir)/rados-classes"/* ) + dostrip -x "${rados_classes[@]#${ED}}" +} + +python_install() { + local CMAKE_USE_DIR="${S}" + DESTDIR="${ED}" cmake_build src/pybind/install + DESTDIR="${ED}" cmake_build src/cephadm/install + + python_optimize +} + +pkg_postinst() { + readme.gentoo_print_elog + tmpfiles_process ${PN}.conf + udev_reload +} + +pkg_postrm() { + udev_reload +} diff --git a/sys-cluster/ceph/files/ceph-19.2.2-py313-1.patch b/sys-cluster/ceph/files/ceph-19.2.2-py313-1.patch new file mode 100644 index 000000000000..b59d7180e5ce --- /dev/null +++ b/sys-cluster/ceph/files/ceph-19.2.2-py313-1.patch @@ -0,0 +1,95 @@ +commit 4cf9b36c66fd64f2dc50f4e1acca3fe93e29b3f2 +Author: Kefu Chai <[email protected]> +Date: Sat Feb 3 19:22:15 2024 +0800 + + mgr: stop using deprecated API to initialize Python + + Py_SetProgramName() is deprecated since CPython 3.11, see + https://docs.python.org/3/c-api/init_config.html . + `Py_InitializeFromConfig()` and friends were introduced by CPython 3.8, + but we still need to support CPython 3.6 which is shipped by CentOS8. + so we have to be backward compatible with the older Python versions. + + so let's use new machinary to initialize the Python interpretor, since + the minimal supported Python version is now CPython 3.9 which comes with + CentOS 9. + + this change addresses following compiling warning: + + ``` + [428/753] Building CXX object src/mgr/CMakeFiles/ceph-mgr.dir/PyModuleRegistry.cc.o + /var/ssd/ceph/src/mgr/PyModuleRegistry.cc: In member function ‘void PyModuleRegistry::init()’: + /var/ssd/ceph/src/mgr/PyModuleRegistry.cc:49:20: warning: ‘void Py_SetProgramName(const wchar_t*)’ is deprecated [-Wdeprecated-declarations] + 49 | Py_SetProgramName(const_cast<wchar_t*>(WCHAR(MGR_PYTHON_EXECUTABLE))); + | ~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + In file included from /usr/include/python3.12/Python.h:94, + from /var/ssd/ceph/src/mgr/PyModule.h:22, + from /var/ssd/ceph/src/mgr/PyModuleRegistry.h:18, + from /var/ssd/ceph/src/mgr/PyModuleRegistry.cc:14: + /usr/include/python3.12/pylifecycle.h:37:38: note: declared here + 37 | Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); + | ^~~~~~~~~~~~~~~~~` + ``` + + Signed-off-by: Kefu Chai <[email protected]> + +diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc +index eb2d2babe75..0d2650d6e59 100644 +--- a/src/mgr/PyModuleRegistry.cc ++++ b/src/mgr/PyModuleRegistry.cc +@@ -14,6 +14,7 @@ + #include "PyModuleRegistry.h" + + #include <filesystem> ++#include <boost/scope_exit.hpp> + + #include "include/stringify.h" + #include "common/errno.h" +@@ -46,21 +47,37 @@ void PyModuleRegistry::init() + + // Set up global python interpreter + #define WCHAR(s) L ## #s +- Py_SetProgramName(const_cast<wchar_t*>(WCHAR(MGR_PYTHON_EXECUTABLE))); +-#undef WCHAR ++ PyConfig py_config; ++ // do not enable isolated mode, otherwise we would not be able to have access ++ // to the site packages. since we cannot import any module before initializing ++ // the interpreter, we would not be able to use "site" module for retrieving ++ // the path to site packager. we import "site" module for retrieving ++ // sitepackages in Python < 3.8 though, this does not apply to the ++ // initialization with PyConfig. ++ PyConfig_InitPythonConfig(&py_config); ++ BOOST_SCOPE_EXIT_ALL(&py_config) { ++ PyConfig_Clear(&py_config); ++ }; ++#if PY_VERSION_HEX >= 0x030b0000 ++ py_config.safe_path = 0; ++#endif ++ py_config.parse_argv = 0; ++ py_config.configure_c_stdio = 0; ++ py_config.install_signal_handlers = 0; ++ py_config.pathconfig_warnings = 0; ++ ++ PyStatus status; ++ status = PyConfig_SetString(&py_config, &py_config.program_name, WCHAR(MGR_PYTHON_EXECUTABLE)); ++ ceph_assertf(!PyStatus_Exception(status), "PyConfig_SetString: %s:%s", status.func, status.err_msg); + // Add more modules + if (g_conf().get_val<bool>("daemonize")) { + PyImport_AppendInittab("ceph_logger", PyModule::init_ceph_logger); + } + PyImport_AppendInittab("ceph_module", PyModule::init_ceph_module); +- Py_InitializeEx(0); +-#if PY_VERSION_HEX < 0x03090000 +- // Let CPython know that we will be calling it back from other +- // threads in future. +- if (! PyEval_ThreadsInitialized()) { +- PyEval_InitThreads(); +- } +-#endif ++ status = Py_InitializeFromConfig(&py_config); ++ ceph_assertf(!PyStatus_Exception(status), "Py_InitializeFromConfig: %s:%s", status.func, status.err_msg); ++#undef WCHAR ++ + // Drop the GIL and remember the main thread state (current + // thread state becomes NULL) + pMainThreadState = PyEval_SaveThread(); diff --git a/sys-cluster/ceph/files/ceph-19.2.2-py313-2.patch b/sys-cluster/ceph/files/ceph-19.2.2-py313-2.patch new file mode 100644 index 000000000000..535d536b4df4 --- /dev/null +++ b/sys-cluster/ceph/files/ceph-19.2.2-py313-2.patch @@ -0,0 +1,61 @@ +commit 07773617f339a2779aa7cf910c0361c937ffe4c2 +Author: Kefu Chai <[email protected]> +Date: Sat Feb 3 19:49:13 2024 +0800 + + mgr: set argv for python in PyModuleRegistry + + before this change, we setup the progname for Python interpreter, + but setup the argv for it in PyModule. and we are using deprecated + API to initialize Python interpreter. + + in this change, let's do this in a single place for better + maintainability. also, take this opportunity, to use the non-deprecated + API to initialize interpreter on Python >= 3.8. + + this silence the warning when compiling ceph-mgr with CPython 3.12: + ``` + /var/ssd/ceph/src/mgr/PyModule.cc: In member function ‘int PyModule::load(PyThreadState*)’: + /var/ssd/ceph/src/mgr/PyModule.cc:363:20: warning: ‘void PySys_SetArgv(int, wchar_t**)’ is deprecated [-Wdeprecated-declarations] + 363 | PySys_SetArgv(1, (wchar_t**)argv); + | ~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~ + In file included from /usr/include/python3.12/Python.h:96, + from /var/ssd/ceph/src/mgr/BaseMgrModule.h:4, + from /var/ssd/ceph/src/mgr/PyModule.cc:14: + /usr/include/python3.12/sysmodule.h:13:38: note: declared here + 13 | Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **); + | ^~~~~~~~~~~~~ + ``` + + Signed-off-by: Kefu Chai <[email protected]> + +diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc +index 0591e2c7145..a723b1ebd55 100644 +--- a/src/mgr/PyModule.cc ++++ b/src/mgr/PyModule.cc +@@ -357,10 +357,6 @@ int PyModule::load(PyThreadState *pMainThreadState) + return -EINVAL; + } else { + pMyThreadState.set(thread_state); +- // Some python modules do not cope with an unpopulated argv, so lets +- // fake one. This step also picks up site-packages into sys.path. +- const wchar_t *argv[] = {L"ceph-mgr"}; +- PySys_SetArgv(1, (wchar_t**)argv); + // Configure sys.path to include mgr_module_path + string paths = (g_conf().get_val<std::string>("mgr_module_path") + ':' + + get_site_packages() + ':'); +diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc +index 0d2650d6e59..8ef532c8662 100644 +--- a/src/mgr/PyModuleRegistry.cc ++++ b/src/mgr/PyModuleRegistry.cc +@@ -69,6 +69,11 @@ void PyModuleRegistry::init() + PyStatus status; + status = PyConfig_SetString(&py_config, &py_config.program_name, WCHAR(MGR_PYTHON_EXECUTABLE)); + ceph_assertf(!PyStatus_Exception(status), "PyConfig_SetString: %s:%s", status.func, status.err_msg); ++ // Some python modules do not cope with an unpopulated argv, so lets ++ // fake one. This step also picks up site-packages into sys.path. ++ const wchar_t* argv[] = {L"ceph-mgr"}; ++ status = PyConfig_SetArgv(&py_config, 1, (wchar_t *const *)argv); ++ ceph_assertf(!PyStatus_Exception(status), "PyConfig_SetArgv: %s:%s", status.func, status.err_msg); + // Add more modules + if (g_conf().get_val<bool>("daemonize")) { + PyImport_AppendInittab("ceph_logger", PyModule::init_ceph_logger); diff --git a/sys-cluster/ceph/files/ceph-19.2.2-py313-3.patch b/sys-cluster/ceph/files/ceph-19.2.2-py313-3.patch new file mode 100644 index 000000000000..753318c2d4d9 --- /dev/null +++ b/sys-cluster/ceph/files/ceph-19.2.2-py313-3.patch @@ -0,0 +1,252 @@ +commit 51a5774aa605f3b976ced47902e15ce450f50339 +Author: Kefu Chai <[email protected]> +Date: Sat Feb 3 21:09:27 2024 +0800 + + mgr: add site package paths in PyModuleRegistry + + before this change, we add the paths of site packages to sys.path + when starting subinterpretors for each of the mgr modules. this + works just fine. but in Python 3.11, it deprecates `PySys_SetPath()` + in favor of PyConfig machinary, which sets the module search paths + in PyConfig, before calling `Py_InitializeFromConfig()`. so, to + set the module search paths with the new machinary, we need to do + this in `PyModuleRegistry`, where we initialize the global Python + interpretor using the new PyConfig machinary. and since we've + switched to the new PyConfig machinary when compiling with Python 3.8 + and up. + + in this change, we set the module search paths in PyModuleRegistry. + because PyConfig imports the site packages by default, and we are + allowed to append a new path to the existing search paths, we just + append the configured `mgr_module_path`. + + this change should silence the compiling warning like: + + ``` + /var/ssd/ceph/src/mgr/PyModule.cc:368:20: warning: ‘void PySys_SetPath(const wchar_t*)’ is deprecated [-Wdeprecated-declarations] + 368 | PySys_SetPath(const_cast<wchar_t*>(sys_path.c_str())); + | ~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + /usr/include/python3.12/sysmodule.h:15:38: note: declared here + 15 | Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetPath(const wchar_t *); + | ^~~~~~~~~~~~~ + ``` + + Fixes https://tracker.ceph.com/issues/66399 + Signed-off-by: Kefu Chai <[email protected]> + +diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc +index a723b1ebd55..cff63ef4a6b 100644 +--- a/src/mgr/PyModule.cc ++++ b/src/mgr/PyModule.cc +@@ -47,7 +47,6 @@ std::string PyModule::mgr_store_prefix = "mgr/"; + + + using std::string; +-using std::wstring; + + // decode a Python exception into a string + std::string handle_pyerror( +@@ -231,72 +230,6 @@ std::pair<int, std::string> PyModuleConfig::set_config( + } + } + +-std::string PyModule::get_site_packages() +-{ +- std::stringstream site_packages; +- +- // CPython doesn't auto-add site-packages dirs to sys.path for us, +- // but it does provide a module that we can ask for them. +- auto site_module = PyImport_ImportModule("site"); +- ceph_assert(site_module); +- +- auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages"); +- if (site_packages_fn != nullptr) { +- auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr); +- ceph_assert(site_packages_list); +- +- auto n = PyList_Size(site_packages_list); +- for (Py_ssize_t i = 0; i < n; ++i) { +- if (i != 0) { +- site_packages << ":"; +- } +- site_packages << PyUnicode_AsUTF8(PyList_GetItem(site_packages_list, i)); +- } +- +- Py_DECREF(site_packages_list); +- Py_DECREF(site_packages_fn); +- } else { +- // Fall back to generating our own site-packages paths by imitating +- // what the standard site.py does. This is annoying but it lets us +- // run inside virtualenvs :-/ +- +- auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages"); +- ceph_assert(site_packages_fn); +- +- auto known_paths = PySet_New(nullptr); +- auto pArgs = PyTuple_Pack(1, known_paths); +- PyObject_CallObject(site_packages_fn, pArgs); +- Py_DECREF(pArgs); +- Py_DECREF(known_paths); +- Py_DECREF(site_packages_fn); +- +- auto sys_module = PyImport_ImportModule("sys"); +- ceph_assert(sys_module); +- auto sys_path = PyObject_GetAttrString(sys_module, "path"); +- ceph_assert(sys_path); +- +- dout(1) << "sys.path:" << dendl; +- auto n = PyList_Size(sys_path); +- bool first = true; +- for (Py_ssize_t i = 0; i < n; ++i) { +- dout(1) << " " << PyUnicode_AsUTF8(PyList_GetItem(sys_path, i)) << dendl; +- if (first) { +- first = false; +- } else { +- site_packages << ":"; +- } +- site_packages << PyUnicode_AsUTF8(PyList_GetItem(sys_path, i)); +- } +- +- Py_DECREF(sys_path); +- Py_DECREF(sys_module); +- } +- +- Py_DECREF(site_module); +- +- return site_packages.str(); +-} +- + PyObject* PyModule::init_ceph_logger() + { + auto py_logger = PyModule_Create(&ceph_logger_module); +@@ -357,13 +290,6 @@ int PyModule::load(PyThreadState *pMainThreadState) + return -EINVAL; + } else { + pMyThreadState.set(thread_state); +- // Configure sys.path to include mgr_module_path +- string paths = (g_conf().get_val<std::string>("mgr_module_path") + ':' + +- get_site_packages() + ':'); +- wstring sys_path(wstring(begin(paths), end(paths)) + Py_GetPath()); +- PySys_SetPath(const_cast<wchar_t*>(sys_path.c_str())); +- dout(10) << "Computed sys.path '" +- << string(begin(sys_path), end(sys_path)) << "'" << dendl; + } + } + // Environment is all good, import the external module +diff --git a/src/mgr/PyModule.h b/src/mgr/PyModule.h +index 8d88ff94c62..177447c2cb3 100644 +--- a/src/mgr/PyModule.h ++++ b/src/mgr/PyModule.h +@@ -51,7 +51,6 @@ class PyModule + mutable ceph::mutex lock = ceph::make_mutex("PyModule::lock"); + private: + const std::string module_name; +- std::string get_site_packages(); + int load_subclass_of(const char* class_name, PyObject** py_class); + + // Did the MgrMap identify this module as one that should run? +diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc +index 8ef532c8662..0eb304e7353 100644 +--- a/src/mgr/PyModuleRegistry.cc ++++ b/src/mgr/PyModuleRegistry.cc +@@ -79,6 +79,15 @@ void PyModuleRegistry::init() + PyImport_AppendInittab("ceph_logger", PyModule::init_ceph_logger); + } + PyImport_AppendInittab("ceph_module", PyModule::init_ceph_module); ++ // Configure sys.path to include mgr_module_path ++ auto pythonpath_env = g_conf().get_val<std::string>("mgr_module_path"); ++ if (const char* pythonpath = getenv("PYTHONPATH")) { ++ pythonpath_env += ":"; ++ pythonpath_env += pythonpath; ++ } ++ status = PyConfig_SetBytesString(&py_config, &py_config.pythonpath_env, pythonpath_env.data()); ++ ceph_assertf(!PyStatus_Exception(status), "PyConfig_SetBytesString: %s:%s", status.func, status.err_msg); ++ dout(10) << "set PYTHONPATH to " << std::quoted(pythonpath_env) << dendl; + status = Py_InitializeFromConfig(&py_config); + ceph_assertf(!PyStatus_Exception(status), "Py_InitializeFromConfig: %s:%s", status.func, status.err_msg); + #undef WCHAR +@@ -239,6 +248,72 @@ void PyModuleRegistry::active_start( + } + } + ++std::string PyModuleRegistry::get_site_packages() ++{ ++ std::stringstream site_packages; ++ ++ // CPython doesn't auto-add site-packages dirs to sys.path for us, ++ // but it does provide a module that we can ask for them. ++ auto site_module = PyImport_ImportModule("site"); ++ ceph_assert(site_module); ++ ++ auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages"); ++ if (site_packages_fn != nullptr) { ++ auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr); ++ ceph_assert(site_packages_list); ++ ++ auto n = PyList_Size(site_packages_list); ++ for (Py_ssize_t i = 0; i < n; ++i) { ++ if (i != 0) { ++ site_packages << ":"; ++ } ++ site_packages << PyUnicode_AsUTF8(PyList_GetItem(site_packages_list, i)); ++ } ++ ++ Py_DECREF(site_packages_list); ++ Py_DECREF(site_packages_fn); ++ } else { ++ // Fall back to generating our own site-packages paths by imitating ++ // what the standard site.py does. This is annoying but it lets us ++ // run inside virtualenvs :-/ ++ ++ auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages"); ++ ceph_assert(site_packages_fn); ++ ++ auto known_paths = PySet_New(nullptr); ++ auto pArgs = PyTuple_Pack(1, known_paths); ++ PyObject_CallObject(site_packages_fn, pArgs); ++ Py_DECREF(pArgs); ++ Py_DECREF(known_paths); ++ Py_DECREF(site_packages_fn); ++ ++ auto sys_module = PyImport_ImportModule("sys"); ++ ceph_assert(sys_module); ++ auto sys_path = PyObject_GetAttrString(sys_module, "path"); ++ ceph_assert(sys_path); ++ ++ dout(1) << "sys.path:" << dendl; ++ auto n = PyList_Size(sys_path); ++ bool first = true; ++ for (Py_ssize_t i = 0; i < n; ++i) { ++ dout(1) << " " << PyUnicode_AsUTF8(PyList_GetItem(sys_path, i)) << dendl; ++ if (first) { ++ first = false; ++ } else { ++ site_packages << ":"; ++ } ++ site_packages << PyUnicode_AsUTF8(PyList_GetItem(sys_path, i)); ++ } ++ ++ Py_DECREF(sys_path); ++ Py_DECREF(sys_module); ++ } ++ ++ Py_DECREF(site_module); ++ ++ return site_packages.str(); ++} ++ + std::vector<std::string> PyModuleRegistry::probe_modules(const std::string &path) const + { + const auto opt = g_conf().get_val<std::string>("mgr_disabled_modules"); +diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h +index 9d6d9c2cdd0..da5bb596c93 100644 +--- a/src/mgr/PyModuleRegistry.h ++++ b/src/mgr/PyModuleRegistry.h +@@ -55,6 +55,7 @@ private: + // before ClusterState exists. + MgrMap mgr_map; + ++ static std::string get_site_packages(); + /** + * Discover python modules from local disk + */
