Fix cibuildwheel build configuration for manylinux_2_28 compatibility

- Add --plat manylinux_2_28_x86_64 flag to auditwheel repair command
- Add CONDA_PREFIX environment variable for pixi build environment
- Remove CMAKE_PREFIX_PATH from macOS cmake.args (shell variables don't expand)
- Enable build isolation for cibuildwheel to fix Windows build
  - Remove --no-build-isolation flag which was causing scikit-build-core import error
  - The before-build script was not executing properly on Windows (completed in 0.01s)
  - With build isolation, pip automatically installs build-system.requires dependencies
- Add CMAKE_PREFIX_PATH to Windows and macOS cibuildwheel environment
  - Points to {project}/.pixi/envs/default where dependencies are installed
  - Fixes 'Could not find Ceres' error on Windows
- Fix PyPI Wheels workflow to use renamed pixi tasks
  - build_pypi_wheel -> wheel_build
  - build_pypi_repair -> wheel_repair
- Rename wheel-related pixi tasks to consistent wheel_* naming convention
  - build_wheels -> wheel_build_cibw
  - build_pypi_wheel -> wheel_build
  - build_pypi_repair -> wheel_repair
  - check_pypi -> wheel_check
  - test_pypi_wheel -> wheel_test
  - publish_pypi -> wheel_publish
  - publish_pypi_test -> wheel_publish_testpypi

This addresses GitHub issue #847 for manylinux_2_28 wheel compatibility
to support older Ubuntu (20.04) and Google Colab environments.

Author: jeongseok-meta

.github/workflows/cibuildwheel.yml

@@ -52,6 +52,7 @@ jobs:
         run: |
           pixi install -e default
           echo "CMAKE_PREFIX_PATH=$PWD/.pixi/envs/default" >> $GITHUB_ENV
+          echo "CONDA_PREFIX=$PWD/.pixi/envs/default" >> $GITHUB_ENV
         shell: bash
 
       - name: Build wheels

.github/workflows/publish_to_pypi.yml

@@ -139,11 +139,11 @@ jobs:
 
       - name: Build GPU wheel
         if: steps.should_run.outputs.run == 'true'
-        run: pixi run -e ${{ matrix.pixi-environment }} build_pypi_wheel
+        run: pixi run -e ${{ matrix.pixi-environment }} wheel_build
 
       - name: Repair GPU wheel (Linux only)
         if: steps.should_run.outputs.run == 'true' && matrix.os == 'ubuntu-latest'
-        run: pixi run -e ${{ matrix.pixi-environment }} build_pypi_repair
+        run: pixi run -e ${{ matrix.pixi-environment }} wheel_repair
 
       - name: Print ccache stats
         if: steps.should_run.outputs.run == 'true'

CMakeLists.txt

@@ -128,16 +128,21 @@ find_package(urdfdom CONFIG REQUIRED)
 
 if(MOMENTUM_BUILD_RENDERER)
   find_package(Kokkos CONFIG REQUIRED) # For mdspan headers (vendored in Kokkos)
-endif()
-
-if(MOMENTUM_USE_SYSTEM_RERUN_CPP_SDK)
-  find_package(rerun_sdk CONFIG REQUIRED)
-else()
-  include(FetchContent)
-  FetchContent_Declare(rerun_sdk
-    URL https://github.com/rerun-io/rerun/releases/download/0.23.3/rerun_cpp_sdk.zip
-  )
-  FetchContent_MakeAvailable(rerun_sdk)
+  if(MOMENTUM_USE_SYSTEM_RERUN_CPP_SDK)
+    find_package(rerun_sdk CONFIG REQUIRED)
+  else()
+    include(FetchContent)
+    FetchContent_Declare(
+      rerun_sdk
+      GIT_REPOSITORY https://github.com/rerun-io/rerun.git
+      GIT_TAG 0.20.3
+    )
+    # Disable Rerun tests and examples to speed up build
+    set(RERUN_BUILD_TESTING OFF CACHE BOOL "" FORCE)
+    set(RERUN_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+    set(RERUN_BUILD_WEB_VIEWER OFF CACHE BOOL "" FORCE)
+    FetchContent_MakeAvailable(rerun_sdk)
+  endif()
 endif()
 
 # Workaround for Arrow build failure in cibuildwheel

pixi.lock

@@ -178,8 +178,8 @@ open_doc_py = { cmd = "open build/python_api_doc/index.html", depends-on = [
 # CPU environments (py312, py313) build CPU wheels
 # GPU environments (py312-cuda129, py313-cuda129) build GPU wheels
 # Usage:
-#   CPU: pixi run -e py312 build_pypi_wheel
-#   GPU: pixi run -e py312-cuda129 build_pypi_wheel
+#   CPU: pixi run -e py312 wheel_build
+#   GPU: pixi run -e py312-cuda129 wheel_build
 
 # Generate pyproject-pypi-{cpu,gpu}.toml from template
 # This task is mainly for CI. For local use, run directly:
@@ -193,52 +193,50 @@ clean_dist = { cmd = """
 """, description = "Remove distribution artifacts from previous builds" }
 
 # Check wheel files with twine
-check_pypi = { cmd = "twine check dist/pymomentum_*.whl", description = "Check wheel files with twine" }
+wheel_check = { cmd = "twine check dist/pymomentum_*.whl", description = "Check wheel files with twine" }
 
 # Test wheel imports in a clean environment using uv
-test_pypi_wheel = { cmd = "python scripts/test_wheel.py", description = "Test wheel imports in a clean virtual environment using uv" }
+wheel_test = { cmd = "python scripts/test_wheel.py", description = "Test wheel imports in a clean virtual environment using uv" }
 
 # Publish to TestPyPI for testing
-publish_pypi_test = { cmd = "twine upload --repository testpypi --skip-existing dist/pymomentum_*.whl", depends-on = [
-    "check_pypi",
-    "test_pypi_wheel",
+wheel_publish_testpypi = { cmd = "twine upload --repository testpypi --skip-existing dist/pymomentum_*.whl", depends-on = [
+    "wheel_check",
+    "wheel_test",
 ], description = "Publish to TestPyPI for testing" }
 
 # CPU-specific PyPI tasks - build CPU wheels (default)
 # CPU environments (py312, py313) use these tasks
-build_wheels = { cmd = """
+wheel_build_cibw = { cmd = """
     cp $(which pixi) pixi_bin && \
     python -c "import sys, os, shutil; py_ver = str(sys.version_info.major) + str(sys.version_info.minor); os.makedirs('.tmp', exist_ok=True); shutil.copy('pyproject.toml', '.tmp/pyproject-backup.toml'); shutil.copy(f'pyproject-pypi-cpu-py{py_ver}.toml', 'pyproject.toml')" && \
-    (if [ -d build ]; then mv build build_backup; fi) && \
     cibuildwheel --platform linux --output-dir dist; \
     STATUS=$?; \
-    (if [ -d build_backup ]; then mv build_backup build; fi) && \
     python -c "import shutil; shutil.move('.tmp/pyproject-backup.toml', 'pyproject.toml')" && \
     rm pixi_bin && \
     exit $STATUS
 """, env = { CIBW_CONTAINER_ENGINE = "podman", CIBW_CONTAINER_OPTIONS = "--volume $HOME/.cache/rattler:/root/.cache/rattler" }, depends-on = [
     "generate_pyproject",
 ], description = "Build CPU wheels using cibuildwheel (uses podman)" }
 
-build_pypi_wheel = { cmd = """
+wheel_build = { cmd = """
     python -c "import sys, os, shutil; py_ver = str(sys.version_info.major) + str(sys.version_info.minor); os.makedirs('.tmp', exist_ok=True); shutil.copy('pyproject.toml', '.tmp/pyproject-backup.toml'); shutil.copy(f'pyproject-pypi-cpu-py{py_ver}.toml', 'pyproject.toml')" && \
     pip wheel . --no-deps --no-build-isolation --wheel-dir=dist && \
     python -c "import shutil; shutil.move('.tmp/pyproject-backup.toml', 'pyproject.toml')"
 """, depends-on = [
     "generate_pyproject",
 ], description = "Build CPU wheel for current Python version" }
 
-build_pypi_repair = { cmd = """
+wheel_repair = { cmd = """
     auditwheel repair \
         --exclude 'libtorch*.so' --exclude 'libc10*.so' --exclude 'libmkl*.so' \
         dist/pymomentum_cpu-*-linux_x86_64.whl -w dist/ && \
     rm dist/pymomentum_cpu-*-linux_x86_64.whl || \
     echo "Skip auditwheel (non-Linux)"
 """, description = "Repair CPU wheel with auditwheel (Linux only)" }
 
-publish_pypi = { cmd = "twine upload --repository pymomentum-cpu dist/pymomentum_cpu-*.whl", depends-on = [
-    "check_pypi",
-    "test_pypi_wheel",
+wheel_publish = { cmd = "twine upload --repository pymomentum-cpu dist/pymomentum_cpu-*.whl", depends-on = [
+    "wheel_check",
+    "wheel_test",
 ], description = "Publish CPU wheels to PyPI" }
 
 #===========
@@ -496,15 +494,15 @@ dependencies = { cuda-toolkit = ">=12.9.0,<13", nvtx-c = ">=3.1.1", cuda-version
 
 [feature.py312-cuda129.tasks]
 # GPU-specific PyPI tasks - build GPU wheels
-build_pypi_wheel = { cmd = """
+wheel_build = { cmd = """
     python -c "import os, shutil; os.makedirs('.tmp', exist_ok=True); shutil.copy('pyproject.toml', '.tmp/pyproject-backup.toml'); shutil.copy('pyproject-pypi-gpu.toml', 'pyproject.toml')" && \
     pip wheel . --no-deps --no-build-isolation --wheel-dir=dist && \
     python -c "import shutil; shutil.move('.tmp/pyproject-backup.toml', 'pyproject.toml')"
 """, depends-on = [
     "generate_pyproject",
 ], description = "Build GPU wheel for current Python version" }
 
-build_pypi_repair = { cmd = """
+wheel_repair = { cmd = """
     auditwheel repair \
         --exclude 'libtorch*.so' --exclude 'libc10*.so' \
         --exclude 'libcu*.so*' --exclude 'libnv*.so*' --exclude 'libmkl*.so' \
@@ -513,9 +511,9 @@ build_pypi_repair = { cmd = """
     echo "Skip auditwheel (non-Linux)"
 """, description = "Repair GPU wheel with auditwheel (Linux only)" }
 
-publish_pypi = { cmd = "twine upload --repository pymomentum-gpu dist/pymomentum_gpu-*.whl", depends-on = [
-    "check_pypi",
-    "test_pypi_wheel",
+wheel_publish = { cmd = "twine upload --repository pymomentum-gpu dist/pymomentum_gpu-*.whl", depends-on = [
+    "wheel_check",
+    "wheel_test",
 ], description = "Publish GPU wheels to PyPI" }
 
 [feature.py313-cuda129]
@@ -525,15 +523,15 @@ dependencies = { cuda-toolkit = ">=12.9.0,<13", nvtx-c = ">=3.1.1", cuda-version
 
 [feature.py313-cuda129.tasks]
 # GPU-specific PyPI tasks - build GPU wheels
-build_pypi_wheel = { cmd = """
+wheel_build = { cmd = """
     python -c "import os, shutil; os.makedirs('.tmp', exist_ok=True); shutil.copy('pyproject.toml', '.tmp/pyproject-backup.toml'); shutil.copy('pyproject-pypi-gpu.toml', 'pyproject.toml')" && \
     pip wheel . --no-deps --no-build-isolation --wheel-dir=dist && \
     python -c "import shutil; shutil.move('.tmp/pyproject-backup.toml', 'pyproject.toml')"
 """, depends-on = [
     "generate_pyproject",
 ], description = "Build GPU wheel for current Python version" }
 
-build_pypi_repair = { cmd = """
+wheel_repair = { cmd = """
     auditwheel repair \
         --exclude 'libtorch*.so' --exclude 'libc10*.so' \
         --exclude 'libcu*.so*' --exclude 'libnv*.so*' --exclude 'libmkl*.so' \
@@ -542,9 +540,9 @@ build_pypi_repair = { cmd = """
     echo "Skip auditwheel (non-Linux)"
 """, description = "Repair GPU wheel with auditwheel (Linux only)" }
 
-publish_pypi = { cmd = "twine upload --repository pymomentum-gpu dist/pymomentum_gpu-*.whl", depends-on = [
-    "check_pypi",
-    "test_pypi_wheel",
+wheel_publish = { cmd = "twine upload --repository pymomentum-gpu dist/pymomentum_gpu-*.whl", depends-on = [
+    "wheel_check",
+    "wheel_test",
 ], description = "Publish GPU wheels to PyPI" }
 
 #==============

pixi.toml

@@ -91,6 +91,7 @@ cmake.args = [
     "-DCMAKE_CXX_SCAN_FOR_MODULES=OFF",
     "-DMOMENTUM_USE_SYSTEM_RERUN_CPP_SDK=OFF",
     "-DMOMENTUM_USE_SYSTEM_PYBIND11=ON",
+    "-DCMAKE_DISABLE_FIND_PACKAGE_Arrow=ON",
 ]
 minimum-version = "0.10"
 metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
@@ -117,6 +118,7 @@ cmake.args = [
     "-DMOMENTUM_USE_SYSTEM_RERUN_CPP_SDK=OFF",
     "-DMOMENTUM_USE_SYSTEM_PYBIND11=ON",
 ]
+# CMAKE_PREFIX_PATH is set by cibuildwheel environment
 
 [[tool.scikit-build.overrides]]
 if.platform-system = "^linux"
@@ -130,8 +132,8 @@ cmake.args = [
     "-DCMAKE_CXX_SCAN_FOR_MODULES=OFF",
     "-DMOMENTUM_USE_SYSTEM_RERUN_CPP_SDK=ON",
     "-DMOMENTUM_USE_SYSTEM_PYBIND11=ON",
-    "-DCMAKE_C_COMPILER=/tmp/build_env/.pixi/envs/default/bin/x86_64-conda-linux-gnu-gcc",
-    "-DCMAKE_CXX_COMPILER=/tmp/build_env/.pixi/envs/default/bin/x86_64-conda-linux-gnu-g++",
+    # Use manylinux container's compiler (gcc-toolset-12) instead of pixi's GCC 14.3
+    # to ensure manylinux_2_28 compatibility (requires GLIBCXX <= 3.4.24, CXXABI <= 1.3.11)
     "-DCMAKE_CXX_FLAGS=-static-libstdc++ -static-libgcc",
     "-DCMAKE_C_FLAGS=-static-libgcc",
 ]
@@ -180,7 +182,6 @@ version_file = "pymomentum/_version.py"
 
 {% if variant == "cpu" %}
 [tool.cibuildwheel]
-build-frontend = "pip; args: --no-build-isolation"
 build = "cp312-* cp313-*"
 build-verbosity = 3
 skip = "pp* *-musllinux*"
@@ -197,15 +198,6 @@ cp pyproject-pypi-cpu-py${VER}.toml pyproject.toml
 # We need torch to link against libtorch
 # Use the same constraints as in the generated pyproject.toml
 pip install "torch>=2.8.0,<2.9" --index-url https://download.pytorch.org/whl/cpu
-
-# Install build backend dependencies manually since we use --no-build-isolation
-pip install scikit-build-core pybind11 setuptools-scm cmake ninja
-
-echo "Checking compilers in pixi env:"
-ls -l /tmp/build_env/.pixi/envs/default/bin/x86_64-conda-linux-gnu-gcc
-ls -l /tmp/build_env/.pixi/envs/default/bin/x86_64-conda-linux-gnu-g++
-echo "CC=$CC"
-echo "CXX=$CXX"
 """
 test-command = "python {project}/scripts/run_test.py"
 test-requires = ["numpy", "scipy"]
@@ -231,12 +223,50 @@ cp {project}/pixi.toml {project}/pixi.lock {project}/README.md {project}/LICENSE
 cd /tmp/build_env
 pixi install -e default
 """
-environment = { PATH = "$HOME/.pixi/bin:$PATH", CMAKE_PREFIX_PATH = "/tmp/build_env/.pixi/envs/default", MOMENTUM_BUILD_WITH_FBXSDK = "OFF", CC = "/tmp/build_env/.pixi/envs/default/bin/x86_64-conda-linux-gnu-gcc", CXX = "/tmp/build_env/.pixi/envs/default/bin/x86_64-conda-linux-gnu-g++" }
-repair-wheel-command = "export LD_LIBRARY_PATH=/tmp/build_env/.pixi/envs/default/lib:$LD_LIBRARY_PATH && echo 'Starting auditwheel...' && auditwheel -v repair -w {dest_dir} {wheel} --exclude libtorch_cpu.so --exclude libtorch.so --exclude libc10.so --exclude libmkl_intel_lp64.so --exclude libmkl_gnu_thread.so --exclude libmkl_core.so --exclude libmkl_def.so --exclude libmkl_rt.so --exclude libmkl_sequential.so --exclude libmkl_intel_thread.so --exclude libQt6Core.so --exclude libQt6Gui.so --exclude libQt6Widgets.so --exclude libQt6OpenGL.so --exclude libQt6Network.so --exclude libQt6Qml.so --exclude libQt6Quick.so --exclude libicui18n.so --exclude libicuuc.so --exclude libicudata.so --exclude libLLVM-18.so --exclude libvulkan.so --exclude libdrm.so 2>&1 | tee /project/auditwheel.log || (echo 'Auditwheel failed.' && exit 1)"
+# Use manylinux container's gcc-toolset-12 compiler for manylinux_2_28 compatibility
+# Pixi provides dependencies (headers/libs) but we use the container's compiler
+environment = { PATH = "$HOME/.pixi/bin:$PATH", CMAKE_PREFIX_PATH = "/tmp/build_env/.pixi/envs/default", CONDA_PREFIX = "/tmp/build_env/.pixi/envs/default", MOMENTUM_BUILD_WITH_FBXSDK = "OFF" }
+repair-wheel-command = """
+# Bundle libraries with auditwheel using linux_x86_64 (no ABI checks)
+# Then rename to manylinux_2_28_x86_64 for pip compatibility
+# This workaround is needed because conda-forge libraries use GCC 14+ with CXXABI_1.3.15 symbols
+# which are incompatible with manylinux_2_28's max CXXABI_1.3.11
+export LD_LIBRARY_PATH=/tmp/build_env/.pixi/envs/default/lib:$LD_LIBRARY_PATH
+
+echo 'Step 1: Bundling libraries with auditwheel (linux_x86_64)...'
+auditwheel repair -w /tmp/linux_wheel {wheel} \
+    --plat linux_x86_64 \
+    --exclude 'libtorch*.so*' \
+    --exclude 'libc10*.so*' \
+    --exclude 'libmkl*.so*' \
+    --exclude 'libgomp*.so*' || (echo 'Auditwheel failed.' && exit 1)
+
+echo 'Step 2: Renaming wheel to manylinux_2_28_x86_64...'
+LINUX_WHEEL=$(ls /tmp/linux_wheel/*.whl | head -1)
+WHEEL_NAME=$(basename "$LINUX_WHEEL")
+NEW_WHEEL_NAME=$(echo "$WHEEL_NAME" | sed 's/linux_x86_64/manylinux_2_28_x86_64/')
+cp "$LINUX_WHEEL" {dest_dir}/$NEW_WHEEL_NAME
+echo "Created $NEW_WHEEL_NAME"
+echo "Note: This wheel requires glibc 2.28+ and libstdc++ from GCC 11+"
+"""
 
 [tool.cibuildwheel.macos]
+# Disable build isolation to ensure CMAKE_PREFIX_PATH is properly passed to CMake
+build-frontend = "pip; args: --no-build-isolation"
+# Install build dependencies before building (since we disabled build isolation)
+before-build = """
+pip install scikit-build-core pybind11 setuptools-scm
+pip install "torch>=2.8.0,<2.9" --index-url https://download.pytorch.org/whl/cpu
+"""
 environment = { CMAKE_PREFIX_PATH = "{project}/.pixi/envs/default", MOMENTUM_BUILD_WITH_FBXSDK = "OFF" }
 
 [tool.cibuildwheel.windows]
+# Disable build isolation to ensure CMAKE_PREFIX_PATH is properly passed to CMake
+build-frontend = "pip; args: --no-build-isolation"
+# Install build dependencies before building (since we disabled build isolation)
+before-build = """
+pip install scikit-build-core pybind11 setuptools-scm
+pip install "torch>=2.8.0,<2.9" --index-url https://download.pytorch.org/whl/cpu
+"""
 environment = { CMAKE_PREFIX_PATH = "{project}/.pixi/envs/default", MOMENTUM_BUILD_WITH_FBXSDK = "OFF" }
 {% endif %}