From 9f71b65cef57d0705c8baf8962b587bc07e398a9 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 18:48:06 +0200 Subject: [PATCH 01/33] Add pymeos cffi build workflow --- .github/workflows/build_pymeos_cffi.yml | 79 +++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/build_pymeos_cffi.yml diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml new file mode 100644 index 00000000..99749ebe --- /dev/null +++ b/.github/workflows/build_pymeos_cffi.yml @@ -0,0 +1,79 @@ +name: Build PyMEOS CFFI + +on: + push: + branches: + - build-actions + create: + tags: + - "pymeos-cffi-[0-9]+.[0-9]+.[0-9]+*" + +jobs: + build: + name: Build PyMEOS CFFI for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-13, macos-14 ] + include: + - ld_path: "/usr/local/lib" + - os: macos-14 + ld_path: "/opt/homebrew/lib" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get dependencies from apt (cache) + uses: awalsh128/cache-apt-pkgs-action@latest + if: runner.os == 'Linux' + with: + packages: build-essential cmake postgresql-server-dev-14 libproj-dev libjson-c-dev libgsl-dev libgeos-dev + version: 1.0 + + - name: Update brew + if: matrix.os == 'macos-13' + # Necessary to avoid issue with macOS runners. See + # https://github.com/actions/runner-images/issues/4020 + run: | + brew reinstall python@3.12 || brew link --overwrite python@3.12 + brew reinstall python@3.11 || brew link --overwrite python@3.11 + brew update + + - name: Get dependencies from homebrew (cache) + uses: tecolicom/actions-use-homebrew-tools@v1 + if: runner.os == 'macOS' + with: + tools: cmake libpq proj json-c gsl geos + + - name: Fetch MEOS sources + env: + BRANCH_NAME: ${{ github.base_ref || github.ref_name }} + run: | + git clone --branch ${{ env.BRANCH_NAME }} --depth 1 https://github.com/MobilityDB/MobilityDB + + - name: Install MEOS + run: | + mkdir MobilityDB/build + cd MobilityDB/build + cmake .. -DMEOS=on + make -j + sudo make install + + # Used to host cibuildwheel + - uses: actions/setup-python@v3 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.17.0 + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + # to supply options, put them in 'env', like: + env: + CIBW_TEST_COMMAND: "from pymeos_cffi import meos_initialize, meos_finalize; meos_initialize('UTC'); meos_finalize()" + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl From 47bbf2155ebbf3436c6ffc52f7192f3781a868ac Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 18:49:43 +0200 Subject: [PATCH 02/33] Always get meos sources from default branch --- .github/workflows/build_pymeos_cffi.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 99749ebe..7a68305e 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -48,10 +48,8 @@ jobs: tools: cmake libpq proj json-c gsl geos - name: Fetch MEOS sources - env: - BRANCH_NAME: ${{ github.base_ref || github.ref_name }} run: | - git clone --branch ${{ env.BRANCH_NAME }} --depth 1 https://github.com/MobilityDB/MobilityDB + git clone --depth 1 https://github.com/MobilityDB/MobilityDB - name: Install MEOS run: | From 42484859f2ab2c68e7748e9f1704469c28747d6c Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 18:52:58 +0200 Subject: [PATCH 03/33] Set working directory of wheel build step --- .github/workflows/build_pymeos_cffi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 7a68305e..4c15b3d1 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -66,6 +66,7 @@ jobs: run: python -m pip install cibuildwheel==2.17.0 - name: Build wheels + working-directory: pymeos_cffi run: python -m cibuildwheel --output-dir wheelhouse # to supply options, put them in 'env', like: env: From 9f48799acc8aff2280a3a48bc5d512f7ea19dc11 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 18:56:12 +0200 Subject: [PATCH 04/33] Add ld_path to exports before building wheels --- .github/workflows/build_pymeos_cffi.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 4c15b3d1..915c10f0 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -67,7 +67,10 @@ jobs: - name: Build wheels working-directory: pymeos_cffi - run: python -m cibuildwheel --output-dir wheelhouse + run: | + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ matrix.ld_path }} + export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_path }} + python -m cibuildwheel --output-dir wheelhouse # to supply options, put them in 'env', like: env: CIBW_TEST_COMMAND: "from pymeos_cffi import meos_initialize, meos_finalize; meos_initialize('UTC'); meos_finalize()" From 5ae066d85e2055342fe4f8cf5a00dfc622db1e1f Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 19:13:33 +0200 Subject: [PATCH 05/33] Install meos in linux wheel build container --- .github/workflows/build_pymeos_cffi.yml | 52 ++++++++++++++++++------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 915c10f0..ec11d0b7 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -9,7 +9,28 @@ on: - "pymeos-cffi-[0-9]+.[0-9]+.[0-9]+*" jobs: - build: + build_sdist: + name: Build PyMEOS CFFI source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + name: Install Python + with: + python-version: 3.8 + + - name: Check metadata + run: "python setup.py check" + - name: Build sdist + run: | + python setup.py sdist + ls -l dist + + - uses: actions/upload-artifact@v4 + with: + path: ./dist/pymeos_cffi-*.tar.gz + + build_wheels: name: Build PyMEOS CFFI for ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: @@ -22,16 +43,6 @@ jobs: ld_path: "/opt/homebrew/lib" steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Get dependencies from apt (cache) - uses: awalsh128/cache-apt-pkgs-action@latest - if: runner.os == 'Linux' - with: - packages: build-essential cmake postgresql-server-dev-14 libproj-dev libjson-c-dev libgsl-dev libgeos-dev - version: 1.0 - - name: Update brew if: matrix.os == 'macos-13' # Necessary to avoid issue with macOS runners. See @@ -48,10 +59,12 @@ jobs: tools: cmake libpq proj json-c gsl geos - name: Fetch MEOS sources + if: runner.os == 'macOS' run: | git clone --depth 1 https://github.com/MobilityDB/MobilityDB - name: Install MEOS + if: runner.os == 'macOS' run: | mkdir MobilityDB/build cd MobilityDB/build @@ -59,8 +72,8 @@ jobs: make -j sudo make install - # Used to host cibuildwheel - - uses: actions/setup-python@v3 + - name: Setup Python + uses: actions/setup-python@v5 - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.17.0 @@ -71,8 +84,19 @@ jobs: export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ matrix.ld_path }} export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_path }} python -m cibuildwheel --output-dir wheelhouse - # to supply options, put them in 'env', like: env: + CIBW_BEFORE_ALL_LINUX: " + yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm + yum -y update + yum -y install gcc gcc-c++ make cmake postgresql13-devel proj81-devel json-c-devel geos39-devel gsl-devel + git clone https://github.com/MobilityDB/MobilityDB + mkdir MobilityDB/build + cd MobilityDB/build + cmake .. -DMEOS=on -DGEOS_INCLUDE_DIR=/usr/geos39/include/ -DGEOS_LIBRARY=/usr/geos39/lib64/libgeos_c.so -DGEOS_CONFIG=/usr/geos39/bin/geos-config -DPROJ_INCLUDE_DIRS=/usr/proj81/include/ -DPROJ_LIBRARIES=/usr/proj81/lib/libproj.so + make -j + make install + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + " CIBW_TEST_COMMAND: "from pymeos_cffi import meos_initialize, meos_finalize; meos_initialize('UTC'); meos_finalize()" - uses: actions/upload-artifact@v4 From 1aff9888ffbe35f35e7238d15fa7014007219fae Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 19:15:10 +0200 Subject: [PATCH 06/33] Set working directory of build_sdist steps --- .github/workflows/build_pymeos_cffi.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index ec11d0b7..07c13de5 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -20,8 +20,10 @@ jobs: python-version: 3.8 - name: Check metadata + working-directory: pymeos_cffi run: "python setup.py check" - name: Build sdist + working-directory: pymeos_cffi run: | python setup.py sdist ls -l dist @@ -101,5 +103,5 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + name: cibw-wheels-${{ matrix.os }} path: ./wheelhouse/*.whl From a24b5ce8458143e72e9de0b2cb215231c8d047b1 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 19:19:30 +0200 Subject: [PATCH 07/33] Add checkout in build_wheels --- .github/workflows/build_pymeos_cffi.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 07c13de5..d9b67c3a 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -13,7 +13,9 @@ jobs: name: Build PyMEOS CFFI source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Install Python with: @@ -22,6 +24,7 @@ jobs: - name: Check metadata working-directory: pymeos_cffi run: "python setup.py check" + - name: Build sdist working-directory: pymeos_cffi run: | @@ -45,6 +48,9 @@ jobs: ld_path: "/opt/homebrew/lib" steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Update brew if: matrix.os == 'macos-13' # Necessary to avoid issue with macOS runners. See From c9831c3d8ab08fa151c62907559409a0c5f7c515 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 19:45:42 +0200 Subject: [PATCH 08/33] Fix cibw linux script --- .github/workflows/build_pymeos_cffi.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index d9b67c3a..24a709fd 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -95,15 +95,15 @@ jobs: env: CIBW_BEFORE_ALL_LINUX: " yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm - yum -y update - yum -y install gcc gcc-c++ make cmake postgresql13-devel proj81-devel json-c-devel geos39-devel gsl-devel - git clone https://github.com/MobilityDB/MobilityDB - mkdir MobilityDB/build - cd MobilityDB/build - cmake .. -DMEOS=on -DGEOS_INCLUDE_DIR=/usr/geos39/include/ -DGEOS_LIBRARY=/usr/geos39/lib64/libgeos_c.so -DGEOS_CONFIG=/usr/geos39/bin/geos-config -DPROJ_INCLUDE_DIRS=/usr/proj81/include/ -DPROJ_LIBRARIES=/usr/proj81/lib/libproj.so - make -j - make install - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + && yum -y update + && yum -y install gcc gcc-c++ make cmake postgresql13-devel proj81-devel json-c-devel geos39-devel gsl-devel + && git clone https://github.com/MobilityDB/MobilityDB + && mkdir MobilityDB/build + && cd MobilityDB/build + && cmake .. -DMEOS=on -DGEOS_INCLUDE_DIR=/usr/geos39/include/ -DGEOS_LIBRARY=/usr/geos39/lib64/libgeos_c.so -DGEOS_CONFIG=/usr/geos39/bin/geos-config -DPROJ_INCLUDE_DIRS=/usr/proj81/include/ -DPROJ_LIBRARIES=/usr/proj81/lib/libproj.so + && make -j + && make install + && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib " CIBW_TEST_COMMAND: "from pymeos_cffi import meos_initialize, meos_finalize; meos_initialize('UTC'); meos_finalize()" From f94a5486ab9d9a728f2926e66db37f3c9f652bb1 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 19:49:26 +0200 Subject: [PATCH 09/33] Fix test command --- .github/workflows/build_pymeos_cffi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 24a709fd..c4b56cd1 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -105,7 +105,7 @@ jobs: && make install && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib " - CIBW_TEST_COMMAND: "from pymeos_cffi import meos_initialize, meos_finalize; meos_initialize('UTC'); meos_finalize()" + CIBW_TEST_COMMAND: "python -c \"from pymeos_cffi import meos_initialize, meos_finalize; meos_initialize('UTC'); meos_finalize()\"" - uses: actions/upload-artifact@v4 with: From 409f12c1fabd481d1b400e3e510e3cc4334d83a7 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 19:51:39 +0200 Subject: [PATCH 10/33] Add log in sdist --- .github/workflows/build_pymeos_cffi.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index c4b56cd1..1fd3ef46 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -28,6 +28,8 @@ jobs: - name: Build sdist working-directory: pymeos_cffi run: | + pwd + ls -l python setup.py sdist ls -l dist From e1bd638f0ee344bd9cc1e9ce1e41ac8e99b1282f Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 19:54:43 +0200 Subject: [PATCH 11/33] Use proper build command for sdist --- .github/workflows/build_pymeos_cffi.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 1fd3ef46..61991ac9 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -21,16 +21,10 @@ jobs: with: python-version: 3.8 - - name: Check metadata - working-directory: pymeos_cffi - run: "python setup.py check" - - name: Build sdist working-directory: pymeos_cffi run: | - pwd - ls -l - python setup.py sdist + python -m build -s ls -l dist - uses: actions/upload-artifact@v4 From 307a7a3d121e23a1277b3c19d30f014f9a430c68 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 19:58:31 +0200 Subject: [PATCH 12/33] Install build module --- .github/workflows/build_pymeos_cffi.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 61991ac9..1db5ce14 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -16,10 +16,16 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - name: Install Python + - name: Install Python + uses: actions/setup-python@v5 with: python-version: 3.8 + cache: "pip" + + - name: Setup pip + run: | + python -m pip install --upgrade pip + python -m pip install build - name: Build sdist working-directory: pymeos_cffi From 40d16f6f9219950bf13b7ee931146e490b3209db Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 20:02:09 +0200 Subject: [PATCH 13/33] Fix artifact path for sdist --- .github/workflows/build_pymeos_cffi.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 1db5ce14..2d0917c9 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -35,9 +35,10 @@ jobs: - uses: actions/upload-artifact@v4 with: - path: ./dist/pymeos_cffi-*.tar.gz + path: ./pymeos_cffi/dist/pymeos_cffi-*.tar.gz build_wheels: + if: false name: Build PyMEOS CFFI for ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: From e36559c2ca99f0f9cb3518771fe83e63d5d37e36 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 20:38:10 +0200 Subject: [PATCH 14/33] Fix wheel artifact folder --- .github/workflows/build_pymeos_cffi.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 2d0917c9..8018f8c0 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -38,7 +38,6 @@ jobs: path: ./pymeos_cffi/dist/pymeos_cffi-*.tar.gz build_wheels: - if: false name: Build PyMEOS CFFI for ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: @@ -113,4 +112,4 @@ jobs: - uses: actions/upload-artifact@v4 with: name: cibw-wheels-${{ matrix.os }} - path: ./wheelhouse/*.whl + path: ./pymeos_cffi/wheelhouse/*.whl From d91a95d0e2789c31d57d441b8f81ece7d91fe11d Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 20:45:34 +0200 Subject: [PATCH 15/33] Skip PyPy builds on linux --- .github/workflows/build_pymeos_cffi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 8018f8c0..826c03fb 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -95,6 +95,7 @@ jobs: export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_path }} python -m cibuildwheel --output-dir wheelhouse env: + CIBW_SKIP: "pp*-manylinux*" # Disable PyPy builds on Linux since shapely has no built distributions for them CIBW_BEFORE_ALL_LINUX: " yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && yum -y update From fb142153c98b8aa5a0f4ad81ce4d2909db09d04c Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 20:52:03 +0200 Subject: [PATCH 16/33] Add logs --- .github/workflows/build_pymeos_cffi.yml | 3 ++- pymeos_cffi/pymeos_cffi/builder/build_pymeos.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 826c03fb..b303d383 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -43,7 +43,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu-latest, macos-13, macos-14 ] +# os: [ ubuntu-latest, macos-13, macos-14 ] + os: [ macos-14 ] include: - ld_path: "/usr/local/lib" - os: macos-14 diff --git a/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py b/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py index 0ffb0d4e..10e997ab 100644 --- a/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py +++ b/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py @@ -43,5 +43,8 @@ def get_include_dirs(): include_dirs=get_include_dirs(), ) # library name, for the linker +print(f"get_library_dirs: {get_library_dirs()}") +print(f"get_include_dirs: {get_include_dirs()}") + if __name__ == "__main__": # not when running with setuptools ffibuilder.compile(verbose=True) From 8b0a835fce827be65ab474cd16bec3fd277a37b6 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 20:55:52 +0200 Subject: [PATCH 17/33] Update library and include dir functions to check for existence of paths rather than os and architecture --- .github/workflows/build_pymeos_cffi.yml | 3 +- .../pymeos_cffi/builder/build_pymeos.py | 32 +++++++------------ 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index b303d383..826c03fb 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -43,8 +43,7 @@ jobs: strategy: fail-fast: false matrix: -# os: [ ubuntu-latest, macos-13, macos-14 ] - os: [ macos-14 ] + os: [ ubuntu-latest, macos-13, macos-14 ] include: - ld_path: "/usr/local/lib" - os: macos-14 diff --git a/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py b/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py index 10e997ab..0f3e235e 100644 --- a/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py +++ b/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py @@ -1,3 +1,4 @@ +import os import platform import sys @@ -12,27 +13,19 @@ def get_library_dirs(): - if sys.platform == "linux": - return ["/usr/local/lib"] - elif sys.platform == "darwin": - if platform.processor() == "arm": - return ["/opt/homebrew/lib"] - else: - return ["/usr/local/lib"] - else: - raise NotImplementedError("Unsupported platform") + paths = [ + "/usr/local/lib", + "/opt/homebrew/lib" + ] + return [path for path in paths if os.path.exists(path)] def get_include_dirs(): - if sys.platform == "linux": - return ["/usr/local/include"] - elif sys.platform == "darwin": - if platform.processor() == "arm": - return ["/opt/homebrew/include"] - else: - return ["/usr/local/include"] - else: - raise NotImplementedError("Unsupported platform") + paths = [ + "/usr/local/include", + "/opt/homebrew/include" + ] + return [path for path in paths if os.path.exists(path)] ffibuilder.set_source( @@ -43,8 +36,5 @@ def get_include_dirs(): include_dirs=get_include_dirs(), ) # library name, for the linker -print(f"get_library_dirs: {get_library_dirs()}") -print(f"get_include_dirs: {get_include_dirs()}") - if __name__ == "__main__": # not when running with setuptools ffibuilder.compile(verbose=True) From 48b740fab49baf184179af53c20e2f7ca0adaf95 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:07:44 +0200 Subject: [PATCH 18/33] Merge macOS MEOS steps. --- .github/workflows/build_pymeos_cffi.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 826c03fb..d3671062 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -68,14 +68,10 @@ jobs: with: tools: cmake libpq proj json-c gsl geos - - name: Fetch MEOS sources - if: runner.os == 'macOS' - run: | - git clone --depth 1 https://github.com/MobilityDB/MobilityDB - - name: Install MEOS if: runner.os == 'macOS' run: | + git clone --depth 1 https://github.com/MobilityDB/MobilityDB mkdir MobilityDB/build cd MobilityDB/build cmake .. -DMEOS=on @@ -96,6 +92,7 @@ jobs: python -m cibuildwheel --output-dir wheelhouse env: CIBW_SKIP: "pp*-manylinux*" # Disable PyPy builds on Linux since shapely has no built distributions for them + CIBW_ARCHS_LINUX: "x86_64 aarch64" CIBW_BEFORE_ALL_LINUX: " yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && yum -y update From ff28d79ff71166d7dc4e922542ee7cc772bd150f Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:08:09 +0200 Subject: [PATCH 19/33] Add QEMU to compile for aarch64 --- .github/workflows/build_pymeos_cffi.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index d3671062..4622fe7f 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -78,6 +78,10 @@ jobs: make -j sudo make install + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + - name: Setup Python uses: actions/setup-python@v5 From b1bc5e70c453c7c4529b1969ad2b38107f42f17a Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:08:59 +0200 Subject: [PATCH 20/33] Add wheel testing job --- .github/workflows/build_pymeos_cffi.yml | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 4622fe7f..bf566d48 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -115,3 +115,51 @@ jobs: with: name: cibw-wheels-${{ matrix.os }} path: ./pymeos_cffi/wheelhouse/*.whl + + test_wheels: + name: Test PyMEOS CFFI wheel - Python ${{ matrix.python-version }} on ${{ matrix.os }} + needs: build_wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + os: [ ubuntu-latest, macos-13, macos-14 ] + exclude: + # Necessary due to issue with macOS runners. See + # https://github.com/actions/setup-python/issues/808 + # Can be removed once this PR is merged: + # https://github.com/actions/python-versions/pull/259 + - os: macos-14 + python-version: "3.8" + - os: macos-14 + python-version: "3.9" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download wheels + uses: actions/download-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }} + path: ./pymeos_cffi_wheels + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install PyMEOS dependencies + run: | + python -m pip install --upgrade pip + pip install -f ./pymeos_cffi_wheels pymeos_cffi + pip install -r pymeos/dev-requirements.txt + + - name: Test PyMEOS with pytest + working-directory: pymeos + run: | + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ matrix.ld_path }} + export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_path }} + pytest From 2ff14253b458b1e98bbbdcfc672144542af54b40 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:09:47 +0200 Subject: [PATCH 21/33] Remove LD exports from pytest step --- .github/workflows/build_pymeos_cffi.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index bf566d48..a527dc19 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -159,7 +159,4 @@ jobs: - name: Test PyMEOS with pytest working-directory: pymeos - run: | - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ matrix.ld_path }} - export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_path }} - pytest + run: pytest From 7d9e19aba44ac107d1da7ef76a8e08d31a904547 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:23:17 +0200 Subject: [PATCH 22/33] Add upload to pypi job --- .github/workflows/build_pymeos_cffi.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index a527dc19..9fbb91ce 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -160,3 +160,20 @@ jobs: - name: Test PyMEOS with pytest working-directory: pymeos run: pytest + + upload_pypi: + name: Upload to PyPI + needs: [ test_wheels, build_sdist ] + runs-on: ubuntu-22.04 + steps: + - uses: actions/download-artifact@v4 + with: + path: ./dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + skip_existing: true From 1f2c8b2d032fc855a6864ce38bfe59ef014e1a2e Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:23:39 +0200 Subject: [PATCH 23/33] Remove aarch64 for linux --- .github/workflows/build_pymeos_cffi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 9fbb91ce..10258f83 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -96,7 +96,7 @@ jobs: python -m cibuildwheel --output-dir wheelhouse env: CIBW_SKIP: "pp*-manylinux*" # Disable PyPy builds on Linux since shapely has no built distributions for them - CIBW_ARCHS_LINUX: "x86_64 aarch64" + CIBW_ARCHS_LINUX: "x86_64" CIBW_BEFORE_ALL_LINUX: " yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && yum -y update From a53985082129ae85b68f90fa3d7622a72e6ce42d Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:31:12 +0200 Subject: [PATCH 24/33] Create GitHub Release job --- .github/workflows/build_pymeos_cffi.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 10258f83..802ad61c 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -177,3 +177,18 @@ jobs: password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository_url: https://test.pypi.org/legacy/ skip_existing: true + + create_release: + name: Create GitHub Release + needs: [ test_wheels, build_sdist ] + runs-on: ubuntu-22.04 + steps: + - uses: actions/download-artifact@v4 + with: + path: ./dist + merge-multiple: true + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + files: ./dist/* \ No newline at end of file From fae8939b3f3759acb932e7d1d72be56ffb4f3168 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:35:08 +0200 Subject: [PATCH 25/33] Disable musllinux builds --- .github/workflows/build_pymeos_cffi.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 802ad61c..989bebf2 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -95,7 +95,10 @@ jobs: export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_path }} python -m cibuildwheel --output-dir wheelhouse env: - CIBW_SKIP: "pp*-manylinux*" # Disable PyPy builds on Linux since shapely has no built distributions for them + # Disable PyPy builds on Linux since shapely has no built distributions for them + # Disable builds on musllinux + # Disable builds in linux architectures other than x86_64 + CIBW_SKIP: "pp*-manylinux* *musllinux*" CIBW_ARCHS_LINUX: "x86_64" CIBW_BEFORE_ALL_LINUX: " yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm From fd9380ba0bc0b5e3c1c106ebb7752f37860da006 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Fri, 12 Apr 2024 22:41:44 +0200 Subject: [PATCH 26/33] Remove QEMU since no cross-compilation is being done anymore --- .github/workflows/build_pymeos_cffi.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 989bebf2..651c8a36 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -78,10 +78,6 @@ jobs: make -j sudo make install - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v2 - - name: Setup Python uses: actions/setup-python@v5 From d4795f96878ed52ef1d7045ead38433f200f00a7 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Sat, 13 Apr 2024 09:39:45 +0200 Subject: [PATCH 27/33] Copy PROJ data in setup.py when PACKAGE_DATA is set to be able to generate self-contained wheels --- pymeos_cffi/setup.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pymeos_cffi/setup.py b/pymeos_cffi/setup.py index 8d234f42..1eb8675e 100644 --- a/pymeos_cffi/setup.py +++ b/pymeos_cffi/setup.py @@ -1,7 +1,36 @@ +import os +import shutil + from setuptools import setup + +def copy_data_tree(source, destination): + try: + shutil.rmtree(destination) + except OSError: + pass + shutil.copytree(source, destination) + + +package_data = [] + +# Conditionally copy PROJ DATA to make self-contained wheels +if os.environ.get("PACKAGE_DATA"): + projdatadir = os.environ.get( + "PROJ_DATA", os.environ.get("PROJ_LIB", "/usr/local/share/proj") + ) + if os.path.exists(projdatadir): + copy_data_tree(projdatadir, "pymeos_cffi/proj_data") + else: + raise FileNotFoundError( + f"PROJ data directory not found at {projdatadir}. " + f"Unable to generate self-contained wheel." + ) + package_data.append("proj_data/*") + setup( packages=["pymeos_cffi", "pymeos_cffi.builder"], setup_requires=["cffi"], + package_data={"pymeos_cffi": package_data}, cffi_modules=["pymeos_cffi/builder/build_pymeos.py:ffibuilder"], ) From 76d99f84f8a0f29a6ea121c010cda05160016985 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Sat, 13 Apr 2024 09:40:28 +0200 Subject: [PATCH 28/33] Pin PROJ versions and set necessary env variables to add proj data to built wheels --- .github/workflows/build_pymeos_cffi.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 651c8a36..b87b525b 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -45,9 +45,18 @@ jobs: matrix: os: [ ubuntu-latest, macos-13, macos-14 ] include: - - ld_path: "/usr/local/lib" + - os: ubuntu-latest + ld_path: "/usr/local/lib" + proj_package: "proj81" + proj_dir: "/usr/proj81" # PROJ 8.1 latest available in manylinux2014 images + - os: macos-13 + ld_path: "/usr/local/lib" + proj_package: "proj@9.4.0" + proj_dir: "/usr/local/Cellar/proj/9.4.0" - os: macos-14 ld_path: "/opt/homebrew/lib" + proj_package: "proj@9.4.0" + proj_dir: "opt/homebrew/Cellar/proj/9.4.0" steps: - name: Checkout @@ -66,7 +75,7 @@ jobs: uses: tecolicom/actions-use-homebrew-tools@v1 if: runner.os == 'macOS' with: - tools: cmake libpq proj json-c gsl geos + tools: cmake libpq ${{ matrix.proj_package }} json-c gsl geos - name: Install MEOS if: runner.os == 'macOS' @@ -89,6 +98,8 @@ jobs: run: | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ matrix.ld_path }} export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_path }} + export PROJ_DATA=${{ matrix.proj_dir }}/share/proj + export PACKAGE_DATA=1 python -m cibuildwheel --output-dir wheelhouse env: # Disable PyPy builds on Linux since shapely has no built distributions for them @@ -99,11 +110,13 @@ jobs: CIBW_BEFORE_ALL_LINUX: " yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && yum -y update - && yum -y install gcc gcc-c++ make cmake postgresql13-devel proj81-devel json-c-devel geos39-devel gsl-devel + && yum -y install gcc gcc-c++ make cmake postgresql13-devel ${{ matrix.proj_package }}-devel json-c-devel geos39-devel gsl-devel + && export PROJ_DATA=${{ matrix.proj_dir }}/share/proj + && export PACKAGE_DATA=1 && git clone https://github.com/MobilityDB/MobilityDB && mkdir MobilityDB/build && cd MobilityDB/build - && cmake .. -DMEOS=on -DGEOS_INCLUDE_DIR=/usr/geos39/include/ -DGEOS_LIBRARY=/usr/geos39/lib64/libgeos_c.so -DGEOS_CONFIG=/usr/geos39/bin/geos-config -DPROJ_INCLUDE_DIRS=/usr/proj81/include/ -DPROJ_LIBRARIES=/usr/proj81/lib/libproj.so + && cmake .. -DMEOS=on -DGEOS_INCLUDE_DIR=/usr/geos39/include/ -DGEOS_LIBRARY=/usr/geos39/lib64/libgeos_c.so -DGEOS_CONFIG=/usr/geos39/bin/geos-config -DPROJ_INCLUDE_DIRS=${{ matrix.proj_dir }}/include/ -DPROJ_LIBRARIES=${{ matrix.proj_dir }}/lib/libproj.so && make -j && make install && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib From 207cbee8615df0b455d629d4f913aee041ebb197 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Sat, 13 Apr 2024 09:47:33 +0200 Subject: [PATCH 29/33] Set PROJ_DATA and PROJ_LIB in meos_initialize if they are undefined Set PROJ_DATA env variables before build wheels step Update PROJ data copy to ignore .tif and .txt files Rename artifacts Fix PROJ_DATA prefix in arm osx Set proj_version dynamically --- .github/workflows/build_pymeos_cffi.yml | 72 +++++++++++-------- .../pymeos_cffi/builder/build_pymeos.py | 14 +--- .../build_pymeos_functions_modifiers.py | 7 ++ .../builder/templates/functions.py | 2 + pymeos_cffi/pymeos_cffi/functions.py | 8 +++ pymeos_cffi/setup.py | 19 ++--- 6 files changed, 71 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index b87b525b..490b6308 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -4,6 +4,7 @@ on: push: branches: - build-actions + - fix-proj-pinning create: tags: - "pymeos-cffi-[0-9]+.[0-9]+.[0-9]+*" @@ -35,6 +36,7 @@ jobs: - uses: actions/upload-artifact@v4 with: + name: pymeos_cffi-sdist path: ./pymeos_cffi/dist/pymeos_cffi-*.tar.gz build_wheels: @@ -45,18 +47,9 @@ jobs: matrix: os: [ ubuntu-latest, macos-13, macos-14 ] include: - - os: ubuntu-latest - ld_path: "/usr/local/lib" - proj_package: "proj81" - proj_dir: "/usr/proj81" # PROJ 8.1 latest available in manylinux2014 images - - os: macos-13 - ld_path: "/usr/local/lib" - proj_package: "proj@9.4.0" - proj_dir: "/usr/local/Cellar/proj/9.4.0" + - ld_prefix: "/usr/local" - os: macos-14 - ld_path: "/opt/homebrew/lib" - proj_package: "proj@9.4.0" - proj_dir: "opt/homebrew/Cellar/proj/9.4.0" + ld_prefix: "/opt/homebrew" steps: - name: Checkout @@ -75,7 +68,15 @@ jobs: uses: tecolicom/actions-use-homebrew-tools@v1 if: runner.os == 'macOS' with: - tools: cmake libpq ${{ matrix.proj_package }} json-c gsl geos + tools: cmake libpq proj json-c gsl geos + + - name: Get PROJ version + id: proj_version + if: runner.os == 'macOS' + run: | + proj_version=$(brew list --versions proj) + proj_version=${proj_version#* } + echo "proj_version=$proj_version" >> $GITHUB_OUTPUT - name: Install MEOS if: runner.os == 'macOS' @@ -93,12 +94,23 @@ jobs: - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.17.0 + - name: Set PROJ_DATA (macOS) + if: runner.os == 'macOS' + run: | + PROJ_DATA=${{ matrix.ld_prefix }}/Cellar/proj/${{ steps.proj_version.outputs.proj_version }}/share/proj + echo "PROJ_DATA=$PROJ_DATA" >> $GITHUB_ENV + + - name: Set PROJ_DATA (Linux) + if: runner.os == 'Linux' + run: | + PROJ_DATA=/usr/proj81/share/proj + echo "PROJ_DATA=$PROJ_DATA" >> $GITHUB_ENV + - name: Build wheels working-directory: pymeos_cffi run: | - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ matrix.ld_path }} - export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_path }} - export PROJ_DATA=${{ matrix.proj_dir }}/share/proj + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ matrix.ld_prefix }}/lib + export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${{ matrix.ld_prefix }}/lib export PACKAGE_DATA=1 python -m cibuildwheel --output-dir wheelhouse env: @@ -107,25 +119,23 @@ jobs: # Disable builds in linux architectures other than x86_64 CIBW_SKIP: "pp*-manylinux* *musllinux*" CIBW_ARCHS_LINUX: "x86_64" - CIBW_BEFORE_ALL_LINUX: " - yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm - && yum -y update - && yum -y install gcc gcc-c++ make cmake postgresql13-devel ${{ matrix.proj_package }}-devel json-c-devel geos39-devel gsl-devel - && export PROJ_DATA=${{ matrix.proj_dir }}/share/proj - && export PACKAGE_DATA=1 - && git clone https://github.com/MobilityDB/MobilityDB - && mkdir MobilityDB/build - && cd MobilityDB/build - && cmake .. -DMEOS=on -DGEOS_INCLUDE_DIR=/usr/geos39/include/ -DGEOS_LIBRARY=/usr/geos39/lib64/libgeos_c.so -DGEOS_CONFIG=/usr/geos39/bin/geos-config -DPROJ_INCLUDE_DIRS=${{ matrix.proj_dir }}/include/ -DPROJ_LIBRARIES=${{ matrix.proj_dir }}/lib/libproj.so - && make -j - && make install - && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib - " + CIBW_ENVIRONMENT_PASS_LINUX: PACKAGE_DATA LD_LIBRARY_PATH PROJ_DATA + CIBW_BEFORE_ALL_LINUX: > + yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && + yum -y update && + yum -y install gcc gcc-c++ make cmake postgresql13-devel proj81-devel json-c-devel geos39-devel gsl-devel && + git clone https://github.com/MobilityDB/MobilityDB && + mkdir MobilityDB/build && + cd MobilityDB/build && + cmake .. -DMEOS=on -DGEOS_INCLUDE_DIR=/usr/geos39/include/ -DGEOS_LIBRARY=/usr/geos39/lib64/libgeos_c.so -DGEOS_CONFIG=/usr/geos39/bin/geos-config -DPROJ_INCLUDE_DIRS=/usr/proj81/include/ -DPROJ_LIBRARIES=/usr/proj81/lib/libproj.so && + make -j && + make install + CIBW_TEST_COMMAND: "python -c \"from pymeos_cffi import meos_initialize, meos_finalize; meos_initialize('UTC'); meos_finalize()\"" - uses: actions/upload-artifact@v4 with: - name: cibw-wheels-${{ matrix.os }} + name: pymeos_cffi-wheels-${{ matrix.os }} path: ./pymeos_cffi/wheelhouse/*.whl test_wheels: @@ -154,7 +164,7 @@ jobs: - name: Download wheels uses: actions/download-artifact@v4 with: - name: cibw-wheels-${{ matrix.os }} + name: pymeos_cffi-wheels-${{ matrix.os }} path: ./pymeos_cffi_wheels - name: Set up Python ${{ matrix.python-version }} diff --git a/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py b/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py index 0f3e235e..92a2bc35 100644 --- a/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py +++ b/pymeos_cffi/pymeos_cffi/builder/build_pymeos.py @@ -1,6 +1,4 @@ import os -import platform -import sys from cffi import FFI @@ -13,18 +11,12 @@ def get_library_dirs(): - paths = [ - "/usr/local/lib", - "/opt/homebrew/lib" - ] + paths = ["/usr/local/lib", "/opt/homebrew/lib"] return [path for path in paths if os.path.exists(path)] def get_include_dirs(): - paths = [ - "/usr/local/include", - "/opt/homebrew/include" - ] + paths = ["/usr/local/include", "/opt/homebrew/include"] return [path for path in paths if os.path.exists(path)] @@ -34,7 +26,7 @@ def get_include_dirs(): libraries=["meos"], library_dirs=get_library_dirs(), include_dirs=get_include_dirs(), -) # library name, for the linker +) if __name__ == "__main__": # not when running with setuptools ffibuilder.compile(verbose=True) diff --git a/pymeos_cffi/pymeos_cffi/builder/build_pymeos_functions_modifiers.py b/pymeos_cffi/pymeos_cffi/builder/build_pymeos_functions_modifiers.py index 2cf1093e..82c7b2c1 100644 --- a/pymeos_cffi/pymeos_cffi/builder/build_pymeos_functions_modifiers.py +++ b/pymeos_cffi/pymeos_cffi/builder/build_pymeos_functions_modifiers.py @@ -42,6 +42,13 @@ def textset_make_modifier(function: str) -> str: def meos_initialize_modifier(_: str) -> str: return """def meos_initialize(tz_str: "Optional[str]") -> None: + + if "PROJ_DATA" not in os.environ and "PROJ_LIB" not in os.environ: + # Assume we are in a wheel and the PROJ data is in the package + proj_dir = os.path.join(os.path.dirname(__file__), "proj_data") + os.environ["PROJ_DATA"] = proj_dir + os.environ["PROJ_LIB"] = proj_dir + tz_str_converted = tz_str.encode('utf-8') if tz_str is not None else _ffi.NULL _lib.meos_initialize(tz_str_converted, _lib.py_error_handler)""" diff --git a/pymeos_cffi/pymeos_cffi/builder/templates/functions.py b/pymeos_cffi/pymeos_cffi/builder/templates/functions.py index 992dfd74..e79c4f88 100644 --- a/pymeos_cffi/pymeos_cffi/builder/templates/functions.py +++ b/pymeos_cffi/pymeos_cffi/builder/templates/functions.py @@ -1,4 +1,6 @@ import logging +import os + from datetime import datetime, timedelta, date from typing import Any, Tuple, Optional, List diff --git a/pymeos_cffi/pymeos_cffi/functions.py b/pymeos_cffi/pymeos_cffi/functions.py index c50e0ff5..4151409a 100644 --- a/pymeos_cffi/pymeos_cffi/functions.py +++ b/pymeos_cffi/pymeos_cffi/functions.py @@ -1,4 +1,6 @@ import logging +import os + from datetime import datetime, timedelta, date from typing import Any, Tuple, Optional, List @@ -203,6 +205,12 @@ def meos_get_intervalstyle() -> str: def meos_initialize(tz_str: "Optional[str]") -> None: + if "PROJ_DATA" not in os.environ and "PROJ_LIB" not in os.environ: + # Assume we are in a wheel and the PROJ data is in the package + proj_dir = os.path.join(os.path.dirname(__file__), "proj_data") + os.environ["PROJ_DATA"] = proj_dir + os.environ["PROJ_LIB"] = proj_dir + tz_str_converted = tz_str.encode("utf-8") if tz_str is not None else _ffi.NULL _lib.meos_initialize(tz_str_converted, _lib.py_error_handler) diff --git a/pymeos_cffi/setup.py b/pymeos_cffi/setup.py index 1eb8675e..7681da29 100644 --- a/pymeos_cffi/setup.py +++ b/pymeos_cffi/setup.py @@ -4,33 +4,34 @@ from setuptools import setup -def copy_data_tree(source, destination): - try: - shutil.rmtree(destination) - except OSError: - pass - shutil.copytree(source, destination) - - package_data = [] # Conditionally copy PROJ DATA to make self-contained wheels if os.environ.get("PACKAGE_DATA"): + print("Copying PROJ data to package data") projdatadir = os.environ.get( "PROJ_DATA", os.environ.get("PROJ_LIB", "/usr/local/share/proj") ) if os.path.exists(projdatadir): - copy_data_tree(projdatadir, "pymeos_cffi/proj_data") + shutil.rmtree("pymeos_cffi/proj_data", ignore_errors=True) + shutil.copytree( + projdatadir, + "pymeos_cffi/proj_data", + ignore=shutil.ignore_patterns("*.txt", "*.tif"), + ) # Don't copy .tiff files and their related .txt files else: raise FileNotFoundError( f"PROJ data directory not found at {projdatadir}. " f"Unable to generate self-contained wheel." ) package_data.append("proj_data/*") +else: + print("Not copying PROJ data to package data") setup( packages=["pymeos_cffi", "pymeos_cffi.builder"], setup_requires=["cffi"], + include_package_data=True, package_data={"pymeos_cffi": package_data}, cffi_modules=["pymeos_cffi/builder/build_pymeos.py:ffibuilder"], ) From 2f33bc4e5b451724b5d4d292a96fff997a524ad7 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Sat, 13 Apr 2024 16:18:26 +0200 Subject: [PATCH 30/33] Build json-c from sources to get version 0.17 instead of default version 0.11 --- .github/workflows/build_pymeos_cffi.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 490b6308..1f80a469 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -123,8 +123,14 @@ jobs: CIBW_BEFORE_ALL_LINUX: > yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && yum -y update && - yum -y install gcc gcc-c++ make cmake postgresql13-devel proj81-devel json-c-devel geos39-devel gsl-devel && - git clone https://github.com/MobilityDB/MobilityDB && + yum -y install gcc gcc-c++ make cmake postgresql13-devel proj81-devel geos39-devel gsl-devel && + git clone --branch json-c-0.17 --depth 1 https://github.com/json-c/json-c && + mkdir json-c-build && + cd json-c-build && + cmake ../json-c && + make && + make install && + git clone --depth 1 https://github.com/MobilityDB/MobilityDB && mkdir MobilityDB/build && cd MobilityDB/build && cmake .. -DMEOS=on -DGEOS_INCLUDE_DIR=/usr/geos39/include/ -DGEOS_LIBRARY=/usr/geos39/lib64/libgeos_c.so -DGEOS_CONFIG=/usr/geos39/bin/geos-config -DPROJ_INCLUDE_DIRS=/usr/proj81/include/ -DPROJ_LIBRARIES=/usr/proj81/lib/libproj.so && From 5a533384c762654a80a30de8419fce8b01d08724 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Sat, 13 Apr 2024 16:28:32 +0200 Subject: [PATCH 31/33] Add json-c path to env for linux --- .github/workflows/build_pymeos_cffi.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 1f80a469..21c3d342 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -100,11 +100,12 @@ jobs: PROJ_DATA=${{ matrix.ld_prefix }}/Cellar/proj/${{ steps.proj_version.outputs.proj_version }}/share/proj echo "PROJ_DATA=$PROJ_DATA" >> $GITHUB_ENV - - name: Set PROJ_DATA (Linux) + - name: Set PROJ_DATA and JSON-C path (Linux) if: runner.os == 'Linux' run: | PROJ_DATA=/usr/proj81/share/proj echo "PROJ_DATA=$PROJ_DATA" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64" >> $GITHUB_ENV - name: Build wheels working-directory: pymeos_cffi From b56031cffc4f0d4e3ef4e6e049ab14d940c20a97 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Sat, 13 Apr 2024 16:58:21 +0200 Subject: [PATCH 32/33] Fix pypi publish action name --- .github/workflows/build_pymeos_cffi.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build_pymeos_cffi.yml b/.github/workflows/build_pymeos_cffi.yml index 21c3d342..6eaf8b2d 100644 --- a/.github/workflows/build_pymeos_cffi.yml +++ b/.github/workflows/build_pymeos_cffi.yml @@ -1,10 +1,6 @@ name: Build PyMEOS CFFI on: - push: - branches: - - build-actions - - fix-proj-pinning create: tags: - "pymeos-cffi-[0-9]+.[0-9]+.[0-9]+*" @@ -200,7 +196,7 @@ jobs: path: ./dist merge-multiple: true - - uses: pypa/gh-action-pypi-publish@v1 + - uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} From 5dd5e0d64d4794be49583b3cdf52e347ba42b6b8 Mon Sep 17 00:00:00 2001 From: Diviloper Date: Sat, 13 Apr 2024 21:23:18 +0200 Subject: [PATCH 33/33] Only set PROJ_DATA if proj_data folder exists (only whould exist on a wheel) --- .../builder/build_pymeos_functions_modifiers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pymeos_cffi/pymeos_cffi/builder/build_pymeos_functions_modifiers.py b/pymeos_cffi/pymeos_cffi/builder/build_pymeos_functions_modifiers.py index 82c7b2c1..7dcd98b5 100644 --- a/pymeos_cffi/pymeos_cffi/builder/build_pymeos_functions_modifiers.py +++ b/pymeos_cffi/pymeos_cffi/builder/build_pymeos_functions_modifiers.py @@ -44,10 +44,11 @@ def meos_initialize_modifier(_: str) -> str: return """def meos_initialize(tz_str: "Optional[str]") -> None: if "PROJ_DATA" not in os.environ and "PROJ_LIB" not in os.environ: - # Assume we are in a wheel and the PROJ data is in the package proj_dir = os.path.join(os.path.dirname(__file__), "proj_data") - os.environ["PROJ_DATA"] = proj_dir - os.environ["PROJ_LIB"] = proj_dir + if os.path.exists(proj_dir): + # Assume we are in a wheel and the PROJ data is in the package + os.environ["PROJ_DATA"] = proj_dir + os.environ["PROJ_LIB"] = proj_dir tz_str_converted = tz_str.encode('utf-8') if tz_str is not None else _ffi.NULL _lib.meos_initialize(tz_str_converted, _lib.py_error_handler)"""