https://github.com/python/cpython/commit/37c35542a9ca18d03f93de5443f8d69bb1be780e commit: 37c35542a9ca18d03f93de5443f8d69bb1be780e branch: main author: Guido Imperiale <[email protected]> committer: kumaraditya303 <[email protected]> date: 2026-02-04T09:58:29+05:30 summary:
gh-143120: pixi builds for free-threading and TSAN (#142872) Co-authored-by: Isuru Fernando <[email protected]> Co-authored-by: Lucas Colley <[email protected]> files: A Tools/pixi-packages/asan/variants.yaml A Tools/pixi-packages/clone-recipe.sh A Tools/pixi-packages/default/variants.yaml A Tools/pixi-packages/freethreading/pixi.toml A Tools/pixi-packages/freethreading/recipe.yaml A Tools/pixi-packages/freethreading/variants.yaml A Tools/pixi-packages/tsan-freethreading/pixi.toml A Tools/pixi-packages/tsan-freethreading/recipe.yaml A Tools/pixi-packages/tsan-freethreading/variants.yaml M Tools/pixi-packages/README.md M Tools/pixi-packages/asan/pixi.toml M Tools/pixi-packages/asan/recipe.yaml M Tools/pixi-packages/build.sh M Tools/pixi-packages/default/pixi.toml M Tools/pixi-packages/default/recipe.yaml diff --git a/Tools/pixi-packages/README.md b/Tools/pixi-packages/README.md index 50c3315ac0e5fc..20bc18f4555fa0 100644 --- a/Tools/pixi-packages/README.md +++ b/Tools/pixi-packages/README.md @@ -1,18 +1,20 @@ # CPython Pixi packages -This directory contains definitions for [Pixi packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) -which can be built from the CPython source code. +This directory contains definitions for [Pixi +packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) which can +be built from the CPython source code. -Downstream developers can make use of these packages by adding them as Git dependencies in a -[Pixi workspace](https://pixi.sh/latest/first_workspace/), like: +Downstream developers can make use of these packages by adding them as Git dependencies +in a [Pixi workspace](https://pixi.sh/latest/first_workspace/), like: ```toml [dependencies] -python = { git = "https://github.com/python/cpython";, subdirectory = "Tools/pixi-packages/asan" } +python.git = "https://github.com/python/cpython"; +python.subdirectory = "Tools/pixi-packages/asan" ``` This is particularly useful when developers need to build CPython from source -(for example, for an ASan-instrumented build), as it does not require any manual +(for example, for an ASan or TSan-instrumented build), as it does not require any manual clone or build steps. Instead, Pixi will automatically handle both the build and installation of the package. @@ -20,17 +22,35 @@ Each package definition is contained in a subdirectory, but they share the build `build.sh` in this directory. Currently defined package variants: - `default` -- `asan`: ASan-instrumented build with `PYTHON_ASAN=1` +- `freethreading` +- `asan`: ASan-instrumented build +- `tsan-freethreading`: TSan-instrumented free-threading build ## Maintenance -- Keep the `version` fields in each `recipe.yaml` up to date with the Python version -- Keep the dependency requirements up to date in each `recipe.yaml` +- Keep the `abi_tag` and `version` fields in each `variants.yaml` up to date with the + Python version - Update `build.sh` for any breaking changes in the `configure` and `make` workflow ## Opportunities for future improvement -- More package variants (such as TSan, UBSan) +- More package variants (such as UBSan) - Support for Windows -- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/4599 -- A workaround can be removed from the build script once https://github.com/prefix-dev/rattler-build/issues/2012 is resolved +- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on + [pixi-build-backends#532](https://github.com/prefix-dev/pixi-build-backends/pull/532) + and [pixi#5248](https://github.com/prefix-dev/pixi/issues/5248) + +## Troubleshooting + +TSan builds may crash on Linux with +``` +FATAL: ThreadSanitizer: unexpected memory mapping 0x7977bd072000-0x7977bd500000 +``` +To fix it, try reducing `mmap_rnd_bits`: + +```bash +$ sudo sysctl vm.mmap_rnd_bits +vm.mmap_rnd_bits = 32 # too high for TSan +$ sudo sysctl vm.mmap_rnd_bits=28 # reduce it +vm.mmap_rnd_bits = 28 +``` diff --git a/Tools/pixi-packages/asan/pixi.toml b/Tools/pixi-packages/asan/pixi.toml index 001ff78fa5d8cb..e3b5673d962659 100644 --- a/Tools/pixi-packages/asan/pixi.toml +++ b/Tools/pixi-packages/asan/pixi.toml @@ -1,6 +1,9 @@ +# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to +# propagate the changes to the other variants. + [workspace] channels = ["https://prefix.dev/conda-forge";] -platforms = ["osx-arm64", "linux-64"] +platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] preview = ["pixi-build"] [package.build.backend] diff --git a/Tools/pixi-packages/asan/recipe.yaml b/Tools/pixi-packages/asan/recipe.yaml index dea88394ad9fe2..30d0d5a2ed2e04 100644 --- a/Tools/pixi-packages/asan/recipe.yaml +++ b/Tools/pixi-packages/asan/recipe.yaml @@ -1,61 +1,92 @@ +# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to +# propagate the changes to the other variants. + context: # Keep up to date - version: "3.15" + freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} -package: +recipe: name: python - version: ${{ version }} source: - path: ../../.. -build: - files: - exclude: - - "*.o" - script: - file: ../build.sh - env: - PYTHON_VARIANT: "asan" - -# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml -requirements: +outputs: +- package: + name: python_abi + version: ${{ version }} + build: + string: "0_${{ abi_tag }}" + requirements: + run_constraints: + - python ${{ version }}.* *_${{ abi_tag }} + +- package: + name: python + version: ${{ version }} build: - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - - make - - pkg-config - # configure script looks for llvm-ar for lto - - if: osx - then: - - llvm-tools - - if: linux - then: - - ld_impl_${{ target_platform }} - - binutils_impl_${{ target_platform }} - - clang-19 - - llvm-tools-19 - - host: - - bzip2 - - sqlite - - liblzma-devel - - zlib - - zstd - - openssl - - readline - - tk - # These two are just to get the headers needed for tk.h, but is unused - - xorg-libx11 - - xorg-xorgproto - - ncurses - - libffi - - if: linux - then: - - ld_impl_${{ target_platform }} - - libuuid - - libmpdec-devel - - expat + string: "0_${{ abi_tag }}" + files: + exclude: + - "*.o" + script: + file: ../build.sh + env: + PYTHON_VARIANT: ${{ variant }} + python: + site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages" + + # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml + requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + # Note that we are not using stdlib arguments which means the packages + # are built for the build settings and are not relocatable to a different + # machine that has a older system version. (eg: macOS/glibc version) + - make + - pkg-config + # configure script looks for llvm-ar for lto + - if: osx + then: + - llvm-tools + + host: + - bzip2 + - sqlite + - liblzma-devel + - zlib + - zstd + - openssl + - readline + - tk + # These two are just to get the headers needed for tk.h, but is unused + - xorg-libx11 + - xorg-xorgproto + - ncurses + - libffi + - if: linux + then: + - libuuid + - libmpdec-devel + - expat + - if: linux and "san" in variant + then: + - libsanitizer + - if: osx and "san" in variant + then: + - libcompiler-rt + + ignore_run_exports: + from_package: + - xorg-libx11 + - xorg-xorgproto + + run_exports: + noarch: + - python + weak: + - python_abi ${{ version }}.* *_${{ abi_tag }} about: homepage: https://www.python.org/ diff --git a/Tools/pixi-packages/asan/variants.yaml b/Tools/pixi-packages/asan/variants.yaml new file mode 100644 index 00000000000000..2404948457e6bb --- /dev/null +++ b/Tools/pixi-packages/asan/variants.yaml @@ -0,0 +1,6 @@ +variant: + - asan +abi_tag: + - asan_cp315 +version: + - 3.15 diff --git a/Tools/pixi-packages/build.sh b/Tools/pixi-packages/build.sh index 120f1d6bb0088a..7e22e6243a5f77 100644 --- a/Tools/pixi-packages/build.sh +++ b/Tools/pixi-packages/build.sh @@ -1,17 +1,36 @@ #!/bin/bash -if [[ "${PYTHON_VARIANT}" == "asan" ]]; then - echo "BUILD TYPE: ASAN" - BUILD_DIR="../build_asan" +echo "PYTHON_VARIANT: ${PYTHON_VARIANT}" + +if [[ "${PYTHON_VARIANT}" == "freethreading" ]]; then + CONFIGURE_EXTRA="--disable-gil" +elif [[ "${PYTHON_VARIANT}" == "asan" ]]; then CONFIGURE_EXTRA="--with-address-sanitizer" - export PYTHON_ASAN="1" export ASAN_OPTIONS="strict_init_order=true" -else - echo "BUILD TYPE: DEFAULT" - BUILD_DIR="../build" +elif [[ "${PYTHON_VARIANT}" == "tsan-freethreading" ]]; then + CONFIGURE_EXTRA="--disable-gil --with-thread-sanitizer" + export TSAN_OPTIONS="suppressions=${SRC_DIR}/Tools/tsan/suppressions_free_threading.txt" +elif [[ "${PYTHON_VARIANT}" == "default" ]]; then CONFIGURE_EXTRA="" +else + echo "Unknown PYTHON_VARIANT: ${PYTHON_VARIANT}" + exit 1 fi +# rattler-build by default set a target of 10.9 +# override it to at least 10.12 +case ${MACOSX_DEPLOYMENT_TARGET:-10.12} in + 10.12|10.13|10.14|10.15|10.16) + ;; + 10.*) + export CPPFLAGS=${CPPFLAGS/-mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}/-mmacosx-version-min=10.12} + export MACOSX_DEPLOYMENT_TARGET=10.12 + ;; + *) + ;; +esac + +BUILD_DIR="../build_${PYTHON_VARIANT}" mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" @@ -23,6 +42,7 @@ else --oldincludedir="${BUILD_PREFIX}/${HOST}/sysroot/usr/include" \ --enable-shared \ --srcdir="${SRC_DIR}" \ + --with-system-expat \ ${CONFIGURE_EXTRA} fi @@ -30,8 +50,3 @@ touch configure-done make -j"${CPU_COUNT}" install ln -sf "${PREFIX}/bin/python3" "${PREFIX}/bin/python" - -# https://github.com/prefix-dev/rattler-build/issues/2012 -if [[ ${OSTYPE} == "darwin"* ]]; then - cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/libclang_rt.asan_osx_dynamic.dylib" -fi diff --git a/Tools/pixi-packages/clone-recipe.sh b/Tools/pixi-packages/clone-recipe.sh new file mode 100755 index 00000000000000..52b2568837c8e1 --- /dev/null +++ b/Tools/pixi-packages/clone-recipe.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Please always only modify default/recipe.yaml and default/pixi.toml and then run this +# script to propagate the changes to the other variants. +set -o errexit +cd "$(dirname "$0")" + +for variant in asan freethreading tsan-freethreading; do + cp -av default/recipe.yaml default/pixi.toml ${variant}/ +done diff --git a/Tools/pixi-packages/default/pixi.toml b/Tools/pixi-packages/default/pixi.toml index 001ff78fa5d8cb..e3b5673d962659 100644 --- a/Tools/pixi-packages/default/pixi.toml +++ b/Tools/pixi-packages/default/pixi.toml @@ -1,6 +1,9 @@ +# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to +# propagate the changes to the other variants. + [workspace] channels = ["https://prefix.dev/conda-forge";] -platforms = ["osx-arm64", "linux-64"] +platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] preview = ["pixi-build"] [package.build.backend] diff --git a/Tools/pixi-packages/default/recipe.yaml b/Tools/pixi-packages/default/recipe.yaml index eeb4052ec3859c..30d0d5a2ed2e04 100644 --- a/Tools/pixi-packages/default/recipe.yaml +++ b/Tools/pixi-packages/default/recipe.yaml @@ -1,59 +1,92 @@ +# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to +# propagate the changes to the other variants. + context: # Keep up to date - version: "3.15" + freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} -package: +recipe: name: python - version: ${{ version }} source: - path: ../../.. -build: - files: - exclude: - - "*.o" - script: - file: ../build.sh +outputs: +- package: + name: python_abi + version: ${{ version }} + build: + string: "0_${{ abi_tag }}" + requirements: + run_constraints: + - python ${{ version }}.* *_${{ abi_tag }} -# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml -requirements: +- package: + name: python + version: ${{ version }} build: - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - - make - - pkg-config - # configure script looks for llvm-ar for lto - - if: osx - then: - - llvm-tools - - if: linux - then: - - ld_impl_${{ target_platform }} - - binutils_impl_${{ target_platform }} - - clang-19 - - llvm-tools-19 - - host: - - bzip2 - - sqlite - - liblzma-devel - - zlib - - zstd - - openssl - - readline - - tk - # These two are just to get the headers needed for tk.h, but is unused - - xorg-libx11 - - xorg-xorgproto - - ncurses - - libffi - - if: linux - then: - - ld_impl_${{ target_platform }} - - libuuid - - libmpdec-devel - - expat + string: "0_${{ abi_tag }}" + files: + exclude: + - "*.o" + script: + file: ../build.sh + env: + PYTHON_VARIANT: ${{ variant }} + python: + site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages" + + # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml + requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + # Note that we are not using stdlib arguments which means the packages + # are built for the build settings and are not relocatable to a different + # machine that has a older system version. (eg: macOS/glibc version) + - make + - pkg-config + # configure script looks for llvm-ar for lto + - if: osx + then: + - llvm-tools + + host: + - bzip2 + - sqlite + - liblzma-devel + - zlib + - zstd + - openssl + - readline + - tk + # These two are just to get the headers needed for tk.h, but is unused + - xorg-libx11 + - xorg-xorgproto + - ncurses + - libffi + - if: linux + then: + - libuuid + - libmpdec-devel + - expat + - if: linux and "san" in variant + then: + - libsanitizer + - if: osx and "san" in variant + then: + - libcompiler-rt + + ignore_run_exports: + from_package: + - xorg-libx11 + - xorg-xorgproto + + run_exports: + noarch: + - python + weak: + - python_abi ${{ version }}.* *_${{ abi_tag }} about: homepage: https://www.python.org/ diff --git a/Tools/pixi-packages/default/variants.yaml b/Tools/pixi-packages/default/variants.yaml new file mode 100644 index 00000000000000..f66e9e7a2226ba --- /dev/null +++ b/Tools/pixi-packages/default/variants.yaml @@ -0,0 +1,6 @@ +variant: + - default +abi_tag: + - cp315 +version: + - 3.15 diff --git a/Tools/pixi-packages/freethreading/pixi.toml b/Tools/pixi-packages/freethreading/pixi.toml new file mode 100644 index 00000000000000..e3b5673d962659 --- /dev/null +++ b/Tools/pixi-packages/freethreading/pixi.toml @@ -0,0 +1,11 @@ +# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to +# propagate the changes to the other variants. + +[workspace] +channels = ["https://prefix.dev/conda-forge";] +platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] +preview = ["pixi-build"] + +[package.build.backend] +name = "pixi-build-rattler-build" +version = "*" diff --git a/Tools/pixi-packages/freethreading/recipe.yaml b/Tools/pixi-packages/freethreading/recipe.yaml new file mode 100644 index 00000000000000..30d0d5a2ed2e04 --- /dev/null +++ b/Tools/pixi-packages/freethreading/recipe.yaml @@ -0,0 +1,94 @@ +# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to +# propagate the changes to the other variants. + +context: + # Keep up to date + freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} + +recipe: + name: python + +source: + - path: ../../.. + +outputs: +- package: + name: python_abi + version: ${{ version }} + build: + string: "0_${{ abi_tag }}" + requirements: + run_constraints: + - python ${{ version }}.* *_${{ abi_tag }} + +- package: + name: python + version: ${{ version }} + build: + string: "0_${{ abi_tag }}" + files: + exclude: + - "*.o" + script: + file: ../build.sh + env: + PYTHON_VARIANT: ${{ variant }} + python: + site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages" + + # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml + requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + # Note that we are not using stdlib arguments which means the packages + # are built for the build settings and are not relocatable to a different + # machine that has a older system version. (eg: macOS/glibc version) + - make + - pkg-config + # configure script looks for llvm-ar for lto + - if: osx + then: + - llvm-tools + + host: + - bzip2 + - sqlite + - liblzma-devel + - zlib + - zstd + - openssl + - readline + - tk + # These two are just to get the headers needed for tk.h, but is unused + - xorg-libx11 + - xorg-xorgproto + - ncurses + - libffi + - if: linux + then: + - libuuid + - libmpdec-devel + - expat + - if: linux and "san" in variant + then: + - libsanitizer + - if: osx and "san" in variant + then: + - libcompiler-rt + + ignore_run_exports: + from_package: + - xorg-libx11 + - xorg-xorgproto + + run_exports: + noarch: + - python + weak: + - python_abi ${{ version }}.* *_${{ abi_tag }} + +about: + homepage: https://www.python.org/ + license: Python-2.0 + license_file: LICENSE diff --git a/Tools/pixi-packages/freethreading/variants.yaml b/Tools/pixi-packages/freethreading/variants.yaml new file mode 100644 index 00000000000000..022833d04c3821 --- /dev/null +++ b/Tools/pixi-packages/freethreading/variants.yaml @@ -0,0 +1,6 @@ +variant: + - freethreading +abi_tag: + - cp315t +version: + - 3.15 diff --git a/Tools/pixi-packages/tsan-freethreading/pixi.toml b/Tools/pixi-packages/tsan-freethreading/pixi.toml new file mode 100644 index 00000000000000..e3b5673d962659 --- /dev/null +++ b/Tools/pixi-packages/tsan-freethreading/pixi.toml @@ -0,0 +1,11 @@ +# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to +# propagate the changes to the other variants. + +[workspace] +channels = ["https://prefix.dev/conda-forge";] +platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] +preview = ["pixi-build"] + +[package.build.backend] +name = "pixi-build-rattler-build" +version = "*" diff --git a/Tools/pixi-packages/tsan-freethreading/recipe.yaml b/Tools/pixi-packages/tsan-freethreading/recipe.yaml new file mode 100644 index 00000000000000..30d0d5a2ed2e04 --- /dev/null +++ b/Tools/pixi-packages/tsan-freethreading/recipe.yaml @@ -0,0 +1,94 @@ +# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to +# propagate the changes to the other variants. + +context: + # Keep up to date + freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} + +recipe: + name: python + +source: + - path: ../../.. + +outputs: +- package: + name: python_abi + version: ${{ version }} + build: + string: "0_${{ abi_tag }}" + requirements: + run_constraints: + - python ${{ version }}.* *_${{ abi_tag }} + +- package: + name: python + version: ${{ version }} + build: + string: "0_${{ abi_tag }}" + files: + exclude: + - "*.o" + script: + file: ../build.sh + env: + PYTHON_VARIANT: ${{ variant }} + python: + site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages" + + # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml + requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + # Note that we are not using stdlib arguments which means the packages + # are built for the build settings and are not relocatable to a different + # machine that has a older system version. (eg: macOS/glibc version) + - make + - pkg-config + # configure script looks for llvm-ar for lto + - if: osx + then: + - llvm-tools + + host: + - bzip2 + - sqlite + - liblzma-devel + - zlib + - zstd + - openssl + - readline + - tk + # These two are just to get the headers needed for tk.h, but is unused + - xorg-libx11 + - xorg-xorgproto + - ncurses + - libffi + - if: linux + then: + - libuuid + - libmpdec-devel + - expat + - if: linux and "san" in variant + then: + - libsanitizer + - if: osx and "san" in variant + then: + - libcompiler-rt + + ignore_run_exports: + from_package: + - xorg-libx11 + - xorg-xorgproto + + run_exports: + noarch: + - python + weak: + - python_abi ${{ version }}.* *_${{ abi_tag }} + +about: + homepage: https://www.python.org/ + license: Python-2.0 + license_file: LICENSE diff --git a/Tools/pixi-packages/tsan-freethreading/variants.yaml b/Tools/pixi-packages/tsan-freethreading/variants.yaml new file mode 100644 index 00000000000000..6ed09fcc9b656b --- /dev/null +++ b/Tools/pixi-packages/tsan-freethreading/variants.yaml @@ -0,0 +1,6 @@ +variant: + - tsan-freethreading +abi_tag: + - tsan_cp315t +version: + - 3.15 _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
