gh-143120: pixi builds for free-threading and TSAN

Tools/pixi-packages/README.md

@@ -12,25 +12,41 @@ python = { git = "https://github.com/python/cpython", subdirectory = "Tools/pixi
 ```
 
 This is particularly useful when developers need to build CPython from source
-(for example, for an ASan-instrumented build), as it does not require any manual
+(for example, for an ASan or TSan-instrumented build), as it does not require any manual
 clone or build steps. Instead, Pixi will automatically handle both the build
 and installation of the package.
 
 Each package definition is contained in a subdirectory, but they share the build script
 `build.sh` in this directory. Currently defined package variants:
 
 - `default`
-- `asan`: ASan-instrumented build with `PYTHON_ASAN=1`
+- `freethreading`
+- `asan`: ASan-instrumented build
+- `tsan_freethreading`: TSan-instrumented free-threading build
 
 ## Maintenance
 
-- Keep the `version` fields in each `recipe.yaml` up to date with the Python version
-- Keep the dependency requirements up to date in each `recipe.yaml`
+- Keep the `abi_tag` and `version` fields in each `pixi.toml` up to date with the Python version
 - Update `build.sh` for any breaking changes in the `configure` and `make` workflow
 
 ## Opportunities for future improvement
 
-- More package variants (such as TSan, UBSan)
+- More package variants (such as UBSan)
 - Support for Windows
 - Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/4599
 - A workaround can be removed from the build script once https://github.com/prefix-dev/rattler-build/issues/2012 is resolved
+
+## Troubleshooting
+
+TSan builds may crash on Linux with
+```
+FATAL: ThreadSanitizer: unexpected memory mapping 0x7977bd072000-0x7977bd500000
+```
+To fix it, try reducing `mmap_rnd_bits`:
+
+```bash
+$ sudo sysctl vm.mmap_rnd_bits
+vm.mmap_rnd_bits = 32  # too high for TSan
+$ sudo sysctl vm.mmap_rnd_bits=28  # reduce it
+vm.mmap_rnd_bits = 28
+```

Tools/pixi-packages/asan/pixi.toml

@@ -0,0 +1 @@
+../default/pixi.toml

Tools/pixi-packages/asan/recipe.yaml

@@ -0,0 +1 @@
+../default/recipe.yaml

Tools/pixi-packages/asan/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - asan
+abi_tag:
+  - asan_cp315
+version:
+  - 3.15

Tools/pixi-packages/build.sh

@@ -1,17 +1,23 @@
 #!/bin/bash
 
-if [[ "${PYTHON_VARIANT}" == "asan" ]]; then
-    echo "BUILD TYPE: ASAN"
-    BUILD_DIR="../build_asan"
+echo "PYTHON_VARIANT: ${PYTHON_VARIANT}"
+
+if [[ "${PYTHON_VARIANT}" == "freethreading" ]]; then
+    CONFIGURE_EXTRA="--disable-gil"
+elif [[ "${PYTHON_VARIANT}" == "asan" ]]; then
     CONFIGURE_EXTRA="--with-address-sanitizer"
-    export PYTHON_ASAN="1"
     export ASAN_OPTIONS="strict_init_order=true"
-else
-    echo "BUILD TYPE: DEFAULT"
-    BUILD_DIR="../build"
+elif [[ "${PYTHON_VARIANT}" == "tsan-freethreading" ]]; then
+    CONFIGURE_EXTRA="--disable-gil --with-thread-sanitizer"
+    export TSAN_OPTIONS="suppressions=${SRC_DIR}/Tools/tsan/suppressions_free_threading.txt"
+elif [[ "${PYTHON_VARIANT}" == "default" ]]; then
     CONFIGURE_EXTRA=""
+else
+    echo "Unknown PYTHON_VARIANT: ${PYTHON_VARIANT}"
+    exit 1
 fi
 
+BUILD_DIR="../build_${PYTHON_VARIANT}"
 mkdir -p "${BUILD_DIR}"
 cd "${BUILD_DIR}"
 
@@ -23,6 +29,7 @@ else
         --oldincludedir="${BUILD_PREFIX}/${HOST}/sysroot/usr/include" \
         --enable-shared \
         --srcdir="${SRC_DIR}" \
+        --with-system-expat \
         ${CONFIGURE_EXTRA}
 fi
 
@@ -33,5 +40,6 @@ ln -sf "${PREFIX}/bin/python3" "${PREFIX}/bin/python"
 
 # https://github.com/prefix-dev/rattler-build/issues/2012
 if [[ ${OSTYPE} == "darwin"* ]]; then
-    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/libclang_rt.asan_osx_dynamic.dylib"
+    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/"
+    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.tsan_osx_dynamic.dylib" "${PREFIX}/lib/"
 fi

Tools/pixi-packages/default/recipe.yaml

@@ -1,59 +1,83 @@
 context:
   # Keep up to date
-  version: "3.15"
+  freethreading_tag: ${{ "t" if "freethreading" in variant else "" }}
 
-package:
+recipe:
   name: python
-  version: ${{ version }}
 
 source:
   - path: ../../..
 
-build:
-  files:
-    exclude:
-      - "*.o"
-  script:
-    file: ../build.sh
+outputs:
+- package:
+    name: python_abi
+    version: ${{ version }}
+  build:
+    string: "0_${{ abi_tag }}"
+  requirements:
+    run_constraints:
+      - python ${{ version }}.* *_${{ abi_tag }}
 
-# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
-requirements:
+- package:
+    name: python
+    version: ${{ version }}
   build:
-    - ${{ compiler('c') }}
-    - ${{ compiler('cxx') }}
-    - make
-    - pkg-config
-    # configure script looks for llvm-ar for lto
-    - if: osx
-      then:
-        - llvm-tools
-    - if: linux
-      then:
-        - ld_impl_${{ target_platform }}
-        - binutils_impl_${{ target_platform }}
-    - clang-19
-    - llvm-tools-19
-
-  host:
-    - bzip2
-    - sqlite
-    - liblzma-devel
-    - zlib
-    - zstd
-    - openssl
-    - readline
-    - tk
-    # These two are just to get the headers needed for tk.h, but is unused
-    - xorg-libx11
-    - xorg-xorgproto
-    - ncurses
-    - libffi
-    - if: linux
-      then:
-        - ld_impl_${{ target_platform }}
-        - libuuid
-    - libmpdec-devel
-    - expat
+    string: "0_${{ abi_tag }}"
+    files:
+      exclude:
+        - "*.o"
+    script:
+      file: ../build.sh
+      env:
+        PYTHON_VARIANT: ${{ variant }}
+    python:
+      site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages"
+
+  # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+  requirements:
+    build:
+      - ${{ compiler('c') }}
+      - ${{ compiler('cxx') }}
+      - make
+      - pkg-config
+      # configure script looks for llvm-ar for lto
+      - if: osx
+        then:
+          - llvm-tools
+
+    host:
+      - bzip2
+      - sqlite
+      - liblzma-devel
+      - zlib
+      - zstd
+      - openssl
+      - readline
+      - tk
+      # These two are just to get the headers needed for tk.h, but is unused
+      - xorg-libx11
+      - xorg-xorgproto
+      - ncurses
+      - libffi
+      - if: linux
+        then:
+          - libuuid
+      - libmpdec-devel
+      - expat
+      - if: c_compiler == "gcc" and "san" in variant
+        then:
+          - libsanitizer
+
+    ignore_run_exports:
+      from_package:
+        - xorg-libx11
+        - xorg-xorgproto
+
+    run_exports:
+      noarch:
+        - python
+      weak:
+        - python_abi ${{ version }}.* *_${{ abi_tag }}
 
 about:
   homepage: https://www.python.org/

Tools/pixi-packages/default/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - default
+abi_tag:
+  - cp315
+version:
+  - 3.15

Tools/pixi-packages/freethreading/pixi.toml

@@ -0,0 +1 @@
+../default/pixi.toml

Tools/pixi-packages/freethreading/recipe.yaml

@@ -0,0 +1 @@
+../default/recipe.yaml

Tools/pixi-packages/freethreading/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - freethreading
+abi_tag:
+  - cp315t
+version:
+  - 3.15

Tools/pixi-packages/tsan-freethreading/pixi.toml

@@ -0,0 +1 @@
+../default/pixi.toml

Tools/pixi-packages/tsan-freethreading/recipe.yaml

@@ -0,0 +1 @@
+../default/recipe.yaml

Tools/pixi-packages/tsan-freethreading/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - tsan-freethreading
+abi_tag:
+  - tsan_cp315t
+version:
+  - 3.15