diff --git a/Tools/pixi-packages/README.md b/Tools/pixi-packages/README.md index 50c3315ac0e5fc..e096cd1ae9ff5f 100644 --- a/Tools/pixi-packages/README.md +++ b/Tools/pixi-packages/README.md @@ -1,18 +1,20 @@ # CPython Pixi packages -This directory contains definitions for [Pixi packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) -which can be built from the CPython source code. +This directory contains definitions for [Pixi +packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) which can +be built from the CPython source code. -Downstream developers can make use of these packages by adding them as Git dependencies in a -[Pixi workspace](https://pixi.sh/latest/first_workspace/), like: +Downstream developers can make use of these packages by adding them as Git dependencies +in a [Pixi workspace](https://pixi.sh/latest/first_workspace/), like: ```toml [dependencies] -python = { git = "https://github.com/python/cpython", subdirectory = "Tools/pixi-packages/asan" } +python.git = "https://github.com/python/cpython" +python.subdirectory = "Tools/pixi-packages/asan" ``` This is particularly useful when developers need to build CPython from source -(for example, for an ASan-instrumented build), as it does not require any manual +(for example, for an ASan or TSan-instrumented build), as it does not require any manual clone or build steps. Instead, Pixi will automatically handle both the build and installation of the package. @@ -20,17 +22,38 @@ Each package definition is contained in a subdirectory, but they share the build `build.sh` in this directory. Currently defined package variants: - `default` -- `asan`: ASan-instrumented build with `PYTHON_ASAN=1` +- `freethreading` +- `asan`: ASan-instrumented build +- `tsan-freethreading`: TSan-instrumented free-threading build ## Maintenance -- Keep the `version` fields in each `recipe.yaml` up to date with the Python version -- Keep the dependency requirements up to date in each `recipe.yaml` +- Keep the `abi_tag` and `version` fields in each `variants.yaml` up to date with the + Python version - Update `build.sh` for any breaking changes in the `configure` and `make` workflow ## Opportunities for future improvement -- More package variants (such as TSan, UBSan) +- More package variants (such as UBSan) - Support for Windows -- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/4599 -- A workaround can be removed from the build script once https://github.com/prefix-dev/rattler-build/issues/2012 is resolved +- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on + [pixi-build-backends#532](https://github.com/prefix-dev/pixi-build-backends/pull/532) + and [pixi#5248](https://github.com/prefix-dev/pixi/issues/5248) +- A workaround can be removed from the build script once + [rattler-build#2012](https://github.com/prefix-dev/rattler-build/issues/2012) is + resolved + +## Troubleshooting + +TSan builds may crash on Linux with +``` +FATAL: ThreadSanitizer: unexpected memory mapping 0x7977bd072000-0x7977bd500000 +``` +To fix it, try reducing `mmap_rnd_bits`: + +```bash +$ sudo sysctl vm.mmap_rnd_bits +vm.mmap_rnd_bits = 32 # too high for TSan +$ sudo sysctl vm.mmap_rnd_bits=28 # reduce it +vm.mmap_rnd_bits = 28 +``` diff --git a/Tools/pixi-packages/asan/pixi.toml b/Tools/pixi-packages/asan/pixi.toml deleted file mode 100644 index 001ff78fa5d8cb..00000000000000 --- a/Tools/pixi-packages/asan/pixi.toml +++ /dev/null @@ -1,8 +0,0 @@ -[workspace] -channels = ["https://prefix.dev/conda-forge"] -platforms = ["osx-arm64", "linux-64"] -preview = ["pixi-build"] - -[package.build.backend] -name = "pixi-build-rattler-build" -version = "*" diff --git a/Tools/pixi-packages/asan/pixi.toml b/Tools/pixi-packages/asan/pixi.toml new file mode 120000 index 00000000000000..3d713eb33ae811 --- /dev/null +++ b/Tools/pixi-packages/asan/pixi.toml @@ -0,0 +1 @@ +../default/pixi.toml \ No newline at end of file diff --git a/Tools/pixi-packages/asan/recipe.yaml b/Tools/pixi-packages/asan/recipe.yaml deleted file mode 100644 index dea88394ad9fe2..00000000000000 --- a/Tools/pixi-packages/asan/recipe.yaml +++ /dev/null @@ -1,63 +0,0 @@ -context: - # Keep up to date - version: "3.15" - -package: - name: python - version: ${{ version }} - -source: - - path: ../../.. - -build: - files: - exclude: - - "*.o" - script: - file: ../build.sh - env: - PYTHON_VARIANT: "asan" - -# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml -requirements: - build: - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - - make - - pkg-config - # configure script looks for llvm-ar for lto - - if: osx - then: - - llvm-tools - - if: linux - then: - - ld_impl_${{ target_platform }} - - binutils_impl_${{ target_platform }} - - clang-19 - - llvm-tools-19 - - host: - - bzip2 - - sqlite - - liblzma-devel - - zlib - - zstd - - openssl - - readline - - tk - # These two are just to get the headers needed for tk.h, but is unused - - xorg-libx11 - - xorg-xorgproto - - ncurses - - libffi - - if: linux - then: - - ld_impl_${{ target_platform }} - - libuuid - - libmpdec-devel - - expat - -about: - homepage: https://www.python.org/ - license: Python-2.0 - license_file: LICENSE diff --git a/Tools/pixi-packages/asan/recipe.yaml b/Tools/pixi-packages/asan/recipe.yaml new file mode 120000 index 00000000000000..90c5c497965560 --- /dev/null +++ b/Tools/pixi-packages/asan/recipe.yaml @@ -0,0 +1 @@ +../default/recipe.yaml \ No newline at end of file diff --git a/Tools/pixi-packages/asan/variants.yaml b/Tools/pixi-packages/asan/variants.yaml new file mode 100644 index 00000000000000..2404948457e6bb --- /dev/null +++ b/Tools/pixi-packages/asan/variants.yaml @@ -0,0 +1,6 @@ +variant: + - asan +abi_tag: + - asan_cp315 +version: + - 3.15 diff --git a/Tools/pixi-packages/build.sh b/Tools/pixi-packages/build.sh index 120f1d6bb0088a..9d00afe311224e 100644 --- a/Tools/pixi-packages/build.sh +++ b/Tools/pixi-packages/build.sh @@ -1,17 +1,23 @@ #!/bin/bash -if [[ "${PYTHON_VARIANT}" == "asan" ]]; then - echo "BUILD TYPE: ASAN" - BUILD_DIR="../build_asan" +echo "PYTHON_VARIANT: ${PYTHON_VARIANT}" + +if [[ "${PYTHON_VARIANT}" == "freethreading" ]]; then + CONFIGURE_EXTRA="--disable-gil" +elif [[ "${PYTHON_VARIANT}" == "asan" ]]; then CONFIGURE_EXTRA="--with-address-sanitizer" - export PYTHON_ASAN="1" export ASAN_OPTIONS="strict_init_order=true" -else - echo "BUILD TYPE: DEFAULT" - BUILD_DIR="../build" +elif [[ "${PYTHON_VARIANT}" == "tsan-freethreading" ]]; then + CONFIGURE_EXTRA="--disable-gil --with-thread-sanitizer" + export TSAN_OPTIONS="suppressions=${SRC_DIR}/Tools/tsan/suppressions_free_threading.txt" +elif [[ "${PYTHON_VARIANT}" == "default" ]]; then CONFIGURE_EXTRA="" +else + echo "Unknown PYTHON_VARIANT: ${PYTHON_VARIANT}" + exit 1 fi +BUILD_DIR="../build_${PYTHON_VARIANT}" mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" @@ -23,6 +29,7 @@ else --oldincludedir="${BUILD_PREFIX}/${HOST}/sysroot/usr/include" \ --enable-shared \ --srcdir="${SRC_DIR}" \ + --with-system-expat \ ${CONFIGURE_EXTRA} fi @@ -33,5 +40,6 @@ ln -sf "${PREFIX}/bin/python3" "${PREFIX}/bin/python" # https://github.com/prefix-dev/rattler-build/issues/2012 if [[ ${OSTYPE} == "darwin"* ]]; then - cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/libclang_rt.asan_osx_dynamic.dylib" + cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/" + cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.tsan_osx_dynamic.dylib" "${PREFIX}/lib/" fi diff --git a/Tools/pixi-packages/default/recipe.yaml b/Tools/pixi-packages/default/recipe.yaml index eeb4052ec3859c..a5609540ae7f68 100644 --- a/Tools/pixi-packages/default/recipe.yaml +++ b/Tools/pixi-packages/default/recipe.yaml @@ -1,59 +1,83 @@ context: # Keep up to date - version: "3.15" + freethreading_tag: ${{ "t" if "freethreading" in variant else "" }} -package: +recipe: name: python - version: ${{ version }} source: - path: ../../.. -build: - files: - exclude: - - "*.o" - script: - file: ../build.sh +outputs: +- package: + name: python_abi + version: ${{ version }} + build: + string: "0_${{ abi_tag }}" + requirements: + run_constraints: + - python ${{ version }}.* *_${{ abi_tag }} -# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml -requirements: +- package: + name: python + version: ${{ version }} build: - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - - make - - pkg-config - # configure script looks for llvm-ar for lto - - if: osx - then: - - llvm-tools - - if: linux - then: - - ld_impl_${{ target_platform }} - - binutils_impl_${{ target_platform }} - - clang-19 - - llvm-tools-19 - - host: - - bzip2 - - sqlite - - liblzma-devel - - zlib - - zstd - - openssl - - readline - - tk - # These two are just to get the headers needed for tk.h, but is unused - - xorg-libx11 - - xorg-xorgproto - - ncurses - - libffi - - if: linux - then: - - ld_impl_${{ target_platform }} - - libuuid - - libmpdec-devel - - expat + string: "0_${{ abi_tag }}" + files: + exclude: + - "*.o" + script: + file: ../build.sh + env: + PYTHON_VARIANT: ${{ variant }} + python: + site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages" + + # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml + requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - make + - pkg-config + # configure script looks for llvm-ar for lto + - if: osx + then: + - llvm-tools + + host: + - bzip2 + - sqlite + - liblzma-devel + - zlib + - zstd + - openssl + - readline + - tk + # These two are just to get the headers needed for tk.h, but is unused + - xorg-libx11 + - xorg-xorgproto + - ncurses + - libffi + - if: linux + then: + - libuuid + - libmpdec-devel + - expat + - if: c_compiler == "gcc" and "san" in variant + then: + - libsanitizer + + ignore_run_exports: + from_package: + - xorg-libx11 + - xorg-xorgproto + + run_exports: + noarch: + - python + weak: + - python_abi ${{ version }}.* *_${{ abi_tag }} about: homepage: https://www.python.org/ diff --git a/Tools/pixi-packages/default/variants.yaml b/Tools/pixi-packages/default/variants.yaml new file mode 100644 index 00000000000000..f66e9e7a2226ba --- /dev/null +++ b/Tools/pixi-packages/default/variants.yaml @@ -0,0 +1,6 @@ +variant: + - default +abi_tag: + - cp315 +version: + - 3.15 diff --git a/Tools/pixi-packages/freethreading/pixi.toml b/Tools/pixi-packages/freethreading/pixi.toml new file mode 120000 index 00000000000000..3d713eb33ae811 --- /dev/null +++ b/Tools/pixi-packages/freethreading/pixi.toml @@ -0,0 +1 @@ +../default/pixi.toml \ No newline at end of file diff --git a/Tools/pixi-packages/freethreading/recipe.yaml b/Tools/pixi-packages/freethreading/recipe.yaml new file mode 120000 index 00000000000000..90c5c497965560 --- /dev/null +++ b/Tools/pixi-packages/freethreading/recipe.yaml @@ -0,0 +1 @@ +../default/recipe.yaml \ No newline at end of file diff --git a/Tools/pixi-packages/freethreading/variants.yaml b/Tools/pixi-packages/freethreading/variants.yaml new file mode 100644 index 00000000000000..022833d04c3821 --- /dev/null +++ b/Tools/pixi-packages/freethreading/variants.yaml @@ -0,0 +1,6 @@ +variant: + - freethreading +abi_tag: + - cp315t +version: + - 3.15 diff --git a/Tools/pixi-packages/tsan-freethreading/pixi.toml b/Tools/pixi-packages/tsan-freethreading/pixi.toml new file mode 120000 index 00000000000000..3d713eb33ae811 --- /dev/null +++ b/Tools/pixi-packages/tsan-freethreading/pixi.toml @@ -0,0 +1 @@ +../default/pixi.toml \ No newline at end of file diff --git a/Tools/pixi-packages/tsan-freethreading/recipe.yaml b/Tools/pixi-packages/tsan-freethreading/recipe.yaml new file mode 120000 index 00000000000000..90c5c497965560 --- /dev/null +++ b/Tools/pixi-packages/tsan-freethreading/recipe.yaml @@ -0,0 +1 @@ +../default/recipe.yaml \ No newline at end of file diff --git a/Tools/pixi-packages/tsan-freethreading/variants.yaml b/Tools/pixi-packages/tsan-freethreading/variants.yaml new file mode 100644 index 00000000000000..6ed09fcc9b656b --- /dev/null +++ b/Tools/pixi-packages/tsan-freethreading/variants.yaml @@ -0,0 +1,6 @@ +variant: + - tsan-freethreading +abi_tag: + - tsan_cp315t +version: + - 3.15