From 921fce8e45741f295187887783ba4dbf9d9564de Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 14:38:23 +0100 Subject: [PATCH 01/39] add python 3.12 to GA testing infrastructure --- .github/workflows/build-and-deploy-on-pypi.yml | 6 +++--- .github/workflows/create-condalock-file.yml | 2 +- .github/workflows/install-from-conda.yml | 4 ++-- .github/workflows/install-from-condalock-file.yml | 2 +- .github/workflows/install-from-pypi.yml | 4 ++-- .github/workflows/install-from-source.yml | 4 ++-- .github/workflows/run-tests-comment.yml | 4 ++-- .github/workflows/run-tests-monitor.yml | 4 ++-- .github/workflows/run-tests.yml | 5 +++-- 9 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-and-deploy-on-pypi.yml b/.github/workflows/build-and-deploy-on-pypi.yml index 449d86b3e4..e698ff1f28 100644 --- a/.github/workflows/build-and-deploy-on-pypi.yml +++ b/.github/workflows/build-and-deploy-on-pypi.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python 3.11 - uses: actions/setup-python@v1 + - name: Set up Python 3.12 + uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" - name: Install pep517 run: >- python -m diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index 6638e296ea..3524ec05d6 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -27,7 +27,7 @@ jobs: with: auto-update-conda: true activate-environment: esmvaltool-fromlock - python-version: "3.11" + python-version: "3.12" miniforge-version: "latest" miniforge-variant: Mambaforge use-mamba: true diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml index 4b9974ebef..e0222e25cf 100644 --- a/.github/workflows/install-from-conda.yml +++ b/.github/workflows/install-from-conda.yml @@ -39,7 +39,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] # fail-fast set to False allows all other tests # in the workflow to run regardless of any fail fail-fast: false @@ -75,7 +75,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml index d6f3d7f5f3..b16cfb8d00 100644 --- a/.github/workflows/install-from-condalock-file.yml +++ b/.github/workflows/install-from-condalock-file.yml @@ -29,7 +29,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/install-from-pypi.yml b/.github/workflows/install-from-pypi.yml index 264e63c41f..7720c4fc67 100644 --- a/.github/workflows/install-from-pypi.yml +++ b/.github/workflows/install-from-pypi.yml @@ -39,7 +39,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] # fail-fast set to False allows all other tests # in the workflow to run regardless of any fail fail-fast: false @@ -77,7 +77,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/install-from-source.yml b/.github/workflows/install-from-source.yml index a01af46437..44188cd3b7 100644 --- a/.github/workflows/install-from-source.yml +++ b/.github/workflows/install-from-source.yml @@ -37,7 +37,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -74,7 +74,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/run-tests-comment.yml b/.github/workflows/run-tests-comment.yml index ccae9db0ee..5e408a9c75 100644 --- a/.github/workflows/run-tests-comment.yml +++ b/.github/workflows/run-tests-comment.yml @@ -14,7 +14,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: Linux Python ${{ matrix.python-version }} if: ${{ github.event.issue.pull_request && github.event.comment.body == '@runGAtests' }} @@ -47,7 +47,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: OSX Python ${{ matrix.python-version }} if: ${{ github.event.issue.pull_request && github.event.comment.body == '@runGAtests' }} diff --git a/.github/workflows/run-tests-monitor.yml b/.github/workflows/run-tests-monitor.yml index 63f4d312ed..721826b652 100644 --- a/.github/workflows/run-tests-monitor.yml +++ b/.github/workflows/run-tests-monitor.yml @@ -22,7 +22,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -55,7 +55,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index aa80283eb9..778c79da08 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,6 +21,7 @@ on: push: branches: - main + - python312 # run the test only if the PR is to main # turn it on if required #pull_request: @@ -39,7 +40,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -71,7 +72,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: From 0467388d29d3ee921d26ad4b8a71a2fe831a6912 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 14:48:19 +0100 Subject: [PATCH 02/39] comment out cartopy --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index cb066cba09..b35a257cdc 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: - nodefaults dependencies: - - cartopy + # - cartopy # currently incompatible with python=3.12 (11 Oct 2023) - cf-units - cftime - compilers From c02215c0db297863eb6a7fe3e0140200ff0decf9 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 14:48:23 +0100 Subject: [PATCH 03/39] comment out cartopy --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 84379ea008..e2fd66578b 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ # Installation dependencies # Use with pip install . to install from source 'install': [ - 'cartopy', + # 'cartopy', # Xpy312 'cf-units', 'dask[array,distributed]', 'dask-jobqueue', From 0f065a526773b436fc6f2615f2295fb796d68a34 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 14:58:22 +0100 Subject: [PATCH 04/39] comment out cfunits --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index b35a257cdc..d690eed51e 100644 --- a/environment.yml +++ b/environment.yml @@ -6,7 +6,7 @@ channels: dependencies: # - cartopy # currently incompatible with python=3.12 (11 Oct 2023) - - cf-units + # - cf-units # currently incompatible with python=3.12 (11 Oct 2023) - cftime - compilers - dask From 410d2678070d936ebffbb4d43e4f6d302c0b5ae0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 14:58:28 +0100 Subject: [PATCH 05/39] comment out cfunits --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e2fd66578b..243f6b9a68 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ # Use with pip install . to install from source 'install': [ # 'cartopy', # Xpy312 - 'cf-units', + # 'cf-units', # Xpy312 'dask[array,distributed]', 'dask-jobqueue', 'esgf-pyclient>=0.3.1', From 901299bff0212ee7bf4ac908511f02a3e03595e6 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 15:03:10 +0100 Subject: [PATCH 06/39] comment out iris --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d690eed51e..3923033f2f 100644 --- a/environment.yml +++ b/environment.yml @@ -20,7 +20,7 @@ dependencies: - geopy - humanfriendly - importlib_metadata # required for Python < 3.10 - - iris >=3.6.0 + # - iris >=3.6.0 # currently incompatible with python=3.12 via cf-units (11 Oct 2023) - iris-esmf-regrid >=0.7.0 - isodate - jinja2 From 43ce46ff12d9080ec4c8fe32016fae46bd2eef1d Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 15:03:15 +0100 Subject: [PATCH 07/39] comment out iris --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 243f6b9a68..cf9f3fd657 100755 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ 'scipy>=1.6', # See the following issue for info on the iris pin below: # https://github.com/ESMValGroup/ESMValTool/issues/3239#issuecomment-1613298587 - 'scitools-iris>=3.4.0', + # 'scitools-iris>=3.4.0', # Xpy312 via cf-units 'shapely>=2.0.0', 'stratify>=0.3', 'yamale', From e649ff8aa5b4b8b71e6cbc1b0c6262d74bcbaae6 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 15:06:04 +0100 Subject: [PATCH 08/39] comment out iris-esmf-regrid --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 3923033f2f..7e4d6b11ad 100644 --- a/environment.yml +++ b/environment.yml @@ -21,7 +21,7 @@ dependencies: - humanfriendly - importlib_metadata # required for Python < 3.10 # - iris >=3.6.0 # currently incompatible with python=3.12 via cf-units (11 Oct 2023) - - iris-esmf-regrid >=0.7.0 + # - iris-esmf-regrid >=0.7.0 # currently incompatible with python=3.12 via cf-units (11 Oct 2023) - isodate - jinja2 - libnetcdf !=4.9.1 # to avoid hdf5 warnings From 924db49e892443bda01d83969fda9786d7c2194c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 15:14:22 +0100 Subject: [PATCH 09/39] pin fiona --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 7e4d6b11ad..c19f1bb809 100644 --- a/environment.yml +++ b/environment.yml @@ -15,7 +15,7 @@ dependencies: - esgf-pyclient >=0.3.1 - esmpy !=8.1.0 - filelock - - fiona + - fiona==1.9.4 # force compat with py312 TODO remove when stable - fire - geopy - humanfriendly From 4717b662276e804dd7bd1485687a602989951629 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 15:34:13 +0100 Subject: [PATCH 10/39] comment out fiona --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index c19f1bb809..ebfa6052b6 100644 --- a/environment.yml +++ b/environment.yml @@ -15,7 +15,7 @@ dependencies: - esgf-pyclient >=0.3.1 - esmpy !=8.1.0 - filelock - - fiona==1.9.4 # force compat with py312 TODO remove when stable + # - fiona # currently incompatible with python=3.12 (11 Oct 2023) - fire - geopy - humanfriendly @@ -37,7 +37,7 @@ dependencies: - psutil - py-cordex - pybtex - - python >=3.9 + - python ==3.12 - python-stratify >=0.3 - pyyaml - requests From 5e1225ec7fd90ff345754b2420f1f7a84601c8f7 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 11 Oct 2023 15:35:18 +0100 Subject: [PATCH 11/39] comment out fiona --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cf9f3fd657..a1194bc82a 100755 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ 'esmf-regrid', 'esmpy!=8.1.0', 'filelock', - 'fiona', + # 'fiona', # Xpy312 'fire', 'geopy', 'humanfriendly', From 4f4e0d1b05c406f5ffb203874d3c200a34a085a5 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 31 Oct 2023 15:28:46 +0000 Subject: [PATCH 12/39] revert commented out packages --- environment.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/environment.yml b/environment.yml index ebfa6052b6..cb066cba09 100644 --- a/environment.yml +++ b/environment.yml @@ -5,8 +5,8 @@ channels: - nodefaults dependencies: - # - cartopy # currently incompatible with python=3.12 (11 Oct 2023) - # - cf-units # currently incompatible with python=3.12 (11 Oct 2023) + - cartopy + - cf-units - cftime - compilers - dask @@ -15,13 +15,13 @@ dependencies: - esgf-pyclient >=0.3.1 - esmpy !=8.1.0 - filelock - # - fiona # currently incompatible with python=3.12 (11 Oct 2023) + - fiona - fire - geopy - humanfriendly - importlib_metadata # required for Python < 3.10 - # - iris >=3.6.0 # currently incompatible with python=3.12 via cf-units (11 Oct 2023) - # - iris-esmf-regrid >=0.7.0 # currently incompatible with python=3.12 via cf-units (11 Oct 2023) + - iris >=3.6.0 + - iris-esmf-regrid >=0.7.0 - isodate - jinja2 - libnetcdf !=4.9.1 # to avoid hdf5 warnings @@ -37,7 +37,7 @@ dependencies: - psutil - py-cordex - pybtex - - python ==3.12 + - python >=3.9 - python-stratify >=0.3 - pyyaml - requests From a7624b698f6eb40ea81f0a0193eefe09a4bd13b5 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 31 Oct 2023 15:28:51 +0000 Subject: [PATCH 13/39] revert commented out packages --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index a1194bc82a..84379ea008 100755 --- a/setup.py +++ b/setup.py @@ -27,15 +27,15 @@ # Installation dependencies # Use with pip install . to install from source 'install': [ - # 'cartopy', # Xpy312 - # 'cf-units', # Xpy312 + 'cartopy', + 'cf-units', 'dask[array,distributed]', 'dask-jobqueue', 'esgf-pyclient>=0.3.1', 'esmf-regrid', 'esmpy!=8.1.0', 'filelock', - # 'fiona', # Xpy312 + 'fiona', 'fire', 'geopy', 'humanfriendly', @@ -58,7 +58,7 @@ 'scipy>=1.6', # See the following issue for info on the iris pin below: # https://github.com/ESMValGroup/ESMValTool/issues/3239#issuecomment-1613298587 - # 'scitools-iris>=3.4.0', # Xpy312 via cf-units + 'scitools-iris>=3.4.0', 'shapely>=2.0.0', 'stratify>=0.3', 'yamale', From d1d8430bdf3755ec6c1b3dc3c460b31be7413821 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 1 Nov 2023 12:36:13 +0000 Subject: [PATCH 14/39] turn off flake8 temporarily --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 778c79da08..bb8940b63c 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -59,7 +59,7 @@ jobs: - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - - run: flake8 + # - run: flake8 # temporary turn off of flake8 since it spits out a number of 3.12-related issues - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail @@ -91,7 +91,7 @@ jobs: - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt - - run: flake8 + # - run: flake8 # temporary turn off of flake8 since it spits out a number of 3.12-related issues - run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail From cadb20e9184404194ff5d7b9c0609fb78c6ba62b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 1 Nov 2023 13:24:04 +0000 Subject: [PATCH 15/39] align mocker test to python 3.12 --- tests/unit/main/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/main/test_main.py b/tests/unit/main/test_main.py index 155c8025d1..546115156a 100644 --- a/tests/unit/main/test_main.py +++ b/tests/unit/main/test_main.py @@ -24,7 +24,7 @@ def test_run_recipe_error(mocker, caplog): print(caplog.text) # Check that the exit status is 1 - assert exit_.called_once_with(1) + exit_.assert_called_once_with(1) # Check that only the RecipeError is logged above DEBUG level errors = [r for r in caplog.records if r.levelname != 'DEBUG'] From 302d5b1539c2b642eb25ff6354492d4c3de05740 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 7 Nov 2023 17:10:15 +0000 Subject: [PATCH 16/39] remake call to flake8 --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index bb8940b63c..778c79da08 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -59,7 +59,7 @@ jobs: - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - # - run: flake8 # temporary turn off of flake8 since it spits out a number of 3.12-related issues + - run: flake8 - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail @@ -91,7 +91,7 @@ jobs: - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt - # - run: flake8 # temporary turn off of flake8 since it spits out a number of 3.12-related issues + - run: flake8 - run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail From 0b1c22223bdfb29db9e910a83c65249fbfcdbf5b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 7 Nov 2023 17:17:05 +0000 Subject: [PATCH 17/39] pin flake8 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index cb066cba09..8c0436a1d1 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,7 @@ dependencies: - sphinx >=6.1.3 - pydata-sphinx-theme # Python packages needed for testing - - flake8 + - flake8 >=6.1.0 # <6 picks up a large number of non-issues in a python=3.12 environment - mypy >=0.990 - pytest >=3.9,!=6.0.0rc1,!=6.0.0 - pytest-cov >=2.10.1 From 1309b495a1266771b5319d86d908da9b113367cc Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 7 Nov 2023 17:17:14 +0000 Subject: [PATCH 18/39] pin flake8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 84379ea008..05f8439616 100755 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ ], # Test dependencies 'test': [ - 'flake8', + 'flake8>=6.1.0', # <6 non-issues in a python=3.12 environment 'pytest>=3.9,!=6.0.0rc1,!=6.0.0', 'pytest-cov>=2.10.1', 'pytest-env', From 244f622cf0a2428c640ec74ae51beda60409b0ae Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 23 Nov 2023 15:47:59 +0000 Subject: [PATCH 19/39] unpin flake8 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 8c0436a1d1..cb066cba09 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,7 @@ dependencies: - sphinx >=6.1.3 - pydata-sphinx-theme # Python packages needed for testing - - flake8 >=6.1.0 # <6 picks up a large number of non-issues in a python=3.12 environment + - flake8 - mypy >=0.990 - pytest >=3.9,!=6.0.0rc1,!=6.0.0 - pytest-cov >=2.10.1 From edd1d7ffe9f550e25e5d1893c85fa8b628981e0e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 23 Nov 2023 15:48:04 +0000 Subject: [PATCH 20/39] unpin flake8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 05f8439616..84379ea008 100755 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ ], # Test dependencies 'test': [ - 'flake8>=6.1.0', # <6 non-issues in a python=3.12 environment + 'flake8', 'pytest>=3.9,!=6.0.0rc1,!=6.0.0', 'pytest-cov>=2.10.1', 'pytest-env', From 7acde282d42ce431c07b8f9bc4f3b572bcafb755 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 23 Nov 2023 16:01:57 +0000 Subject: [PATCH 21/39] recomment out flake8 runs --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 778c79da08..b195a90011 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -59,7 +59,7 @@ jobs: - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - - run: flake8 + # - run: flake8 - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail @@ -91,7 +91,7 @@ jobs: - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt - - run: flake8 + # - run: flake8 - run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail From fb1b4d62cb710b8ef49002e27bd3996f947f1ec1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 12 Jan 2024 15:48:54 +0000 Subject: [PATCH 22/39] unpin upper Python --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 93582e968c..68171cc807 100644 --- a/environment.yml +++ b/environment.yml @@ -37,7 +37,7 @@ dependencies: - psutil - py-cordex - pybtex - - python >=3.9,<3.12 + - python >=3.9 - python-stratify >=0.3 - pyyaml - requests From f79c4eca196d2fabfae86be658adc02f9be451bb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 12 Jan 2024 16:24:41 +0000 Subject: [PATCH 23/39] retrigger flake8 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b195a90011..ec5aabe1d8 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -59,7 +59,7 @@ jobs: - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - # - run: flake8 + - run: flake8 - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail From 7a025e0a24e550d8f959085263c9036e90ef8f0f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 12 Jan 2024 16:43:58 +0000 Subject: [PATCH 24/39] ignore prospector for now --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 68171cc807..964c023883 100644 --- a/environment.yml +++ b/environment.yml @@ -70,7 +70,7 @@ dependencies: - docformatter - isort - pre-commit - - prospector >=1.9.0 + # - prospector >=1.9.0 # Not on conda forge - vprof - yamllint - yapf From 501e59d41c04395284b41a3cb330c18e4d5a2d79 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 15 Jan 2024 14:12:42 +0000 Subject: [PATCH 25/39] pin flake8 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 964c023883..1c56349732 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,7 @@ dependencies: - sphinx >=6.1.3 - pydata-sphinx-theme # Python packages needed for testing - - flake8 + - flake8 >=7.0.0 # not to pick up E231 - mypy >=0.990 - pytest >=3.9,!=6.0.0rc1,!=6.0.0 - pytest-cov >=2.10.1 From 4e04603c6f46268eecefb2500f06e5802c55de15 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 15 Jan 2024 14:12:47 +0000 Subject: [PATCH 26/39] pin flake8 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f8c3e28550..fc9226ac83 100755 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ ], # Test dependencies 'test': [ - 'flake8', + 'flake8>=7.0.0', # not to pick up E231 'pytest>=3.9,!=6.0.0rc1,!=6.0.0', 'pytest-cov>=2.10.1', 'pytest-env', @@ -95,7 +95,7 @@ 'docformatter', 'isort', 'pre-commit', - 'prospector[with_pyroma]>=1.9.0', + # 'prospector[with_pyroma]>=1.9.0', 'vprof', 'yamllint', 'yapf', From 57fcea4777803f16611b61ae4abf3a4f43c7ab2a Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 15 Jan 2024 14:29:52 +0000 Subject: [PATCH 27/39] fix new flake8 actual error --- esmvalcore/_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 04200371cd..2b785bc8d6 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -566,7 +566,7 @@ def _run(self, input_files): returncode = None - with resource_usage_logger(process.pid, self.resource_log),\ + with resource_usage_logger(process.pid, self.resource_log), \ open(self.log, 'ab') as log: last_line = [''] while returncode is None: From 909bacecd9adf4f81ea0c0d203c2de7dedb7f4e6 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 5 Apr 2024 16:40:57 +0100 Subject: [PATCH 28/39] inspect environment --- .github/workflows/run-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ec5aabe1d8..3615161ac6 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -59,6 +59,7 @@ jobs: - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt + - run: conda list - run: flake8 - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts From cbc9d2cf34ea5e5b42d05c45d7de97df5d58d8c2 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Apr 2024 14:40:19 +0100 Subject: [PATCH 29/39] fix inadvertent change --- esmvalcore/_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 2b785bc8d6..04200371cd 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -566,7 +566,7 @@ def _run(self, input_files): returncode = None - with resource_usage_logger(process.pid, self.resource_log), \ + with resource_usage_logger(process.pid, self.resource_log),\ open(self.log, 'ab') as log: last_line = [''] while returncode is None: From 1566c2b7c9867f080d609c85ae9744ae5083c2e0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Apr 2024 15:02:13 +0100 Subject: [PATCH 30/39] readd space after comma - flak 7 is complaining --- esmvalcore/_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 04200371cd..2b785bc8d6 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -566,7 +566,7 @@ def _run(self, input_files): returncode = None - with resource_usage_logger(process.pid, self.resource_log),\ + with resource_usage_logger(process.pid, self.resource_log), \ open(self.log, 'ab') as log: last_line = [''] while returncode is None: From dcf2dc692e9644dfba0a66c1396bb8738c3238fb Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 28 May 2024 08:21:41 +0200 Subject: [PATCH 31/39] Attempt at supporting Python 3.12 --- environment.yml | 8 ++++++-- esmvalcore/_task.py | 2 +- setup.py | 3 ++- tests/unit/main/test_main.py | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/environment.yml b/environment.yml index 4cfed229c3..0f7c13545a 100644 --- a/environment.yml +++ b/environment.yml @@ -36,7 +36,7 @@ dependencies: - psutil - py-cordex - pybtex - - python >=3.9,<3.12 + - python >=3.9 - python-stratify >=0.3 - pyyaml - requests @@ -50,7 +50,6 @@ dependencies: - sphinx >=6.1.3 - pydata-sphinx-theme # Python packages needed for testing - - flake8 - mypy >=0.990 - pytest >=3.9,!=6.0.0rc1,!=6.0.0 - pytest-cov >=2.10.1 @@ -58,6 +57,7 @@ dependencies: - pytest-html !=2.1.0 - pytest-metadata >=1.5.1 - pytest-mock + - pytest-mypy - pytest-xdist # Not on conda-forge - ESMValTool_sample_data==0.0.3 # Still for testing, MyPy library stubs @@ -69,6 +69,10 @@ dependencies: - isort - pre-commit - prospector >=1.9.0 + - pyroma # Not on conda forge - vprof - yamllint - yapf + - pip: + - flake8>=7 + - ESMValTool_sample_data diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 04200371cd..2b785bc8d6 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -566,7 +566,7 @@ def _run(self, input_files): returncode = None - with resource_usage_logger(process.pid, self.resource_log),\ + with resource_usage_logger(process.pid, self.resource_log), \ open(self.log, 'ab') as log: last_line = [''] while returncode is None: diff --git a/setup.py b/setup.py index 0a7bf9ab97..95f426a282 100755 --- a/setup.py +++ b/setup.py @@ -94,8 +94,9 @@ 'codespell', 'docformatter', 'isort', + 'flake8>=7', 'pre-commit', - 'prospector[with_pyroma]>=1.9.0', + # 'prospector[with_pyroma]>=1.9.0', 'vprof', 'yamllint', 'yapf', diff --git a/tests/unit/main/test_main.py b/tests/unit/main/test_main.py index 155c8025d1..546115156a 100644 --- a/tests/unit/main/test_main.py +++ b/tests/unit/main/test_main.py @@ -24,7 +24,7 @@ def test_run_recipe_error(mocker, caplog): print(caplog.text) # Check that the exit status is 1 - assert exit_.called_once_with(1) + exit_.assert_called_once_with(1) # Check that only the RecipeError is logged above DEBUG level errors = [r for r in caplog.records if r.levelname != 'DEBUG'] From 4aa944ee480b81aa2dae698c377588911defc9ae Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 28 May 2024 12:15:53 +0100 Subject: [PATCH 32/39] reinstate prospector in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4329ebddb2..968404f216 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ 'isort', 'flake8>=7', 'pre-commit', - # 'prospector[with_pyroma]>=1.9.0', + 'prospector[with_pyroma]>=1.9.0', 'vprof', 'yamllint', 'yapf', From 487cdcd52bb4643377bab59ddbaa3dffba7fadb4 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 28 May 2024 12:22:00 +0100 Subject: [PATCH 33/39] remove prospector pin --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 968404f216..700e47c33c 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ 'isort', 'flake8>=7', 'pre-commit', - 'prospector[with_pyroma]>=1.9.0', + 'prospector', 'vprof', 'yamllint', 'yapf', From 09ae65689fad8d1b51ddbb744f16391e70af7509 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 28 May 2024 12:40:18 +0100 Subject: [PATCH 34/39] Remove prospector from setup.py altogether --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 700e47c33c..f89385fbd2 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ 'isort', 'flake8>=7', 'pre-commit', - 'prospector', + # 'prospector', # pip installs old v0.12, keep conda-forge one 'vprof', 'yamllint', 'yapf', From a30df92dd173964791b496791a87fdf28fafe08c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 28 May 2024 12:41:50 +0100 Subject: [PATCH 35/39] pop a conda list examine env before pip install --- .github/workflows/run-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f3f8fca340..4720a62396 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -58,6 +58,7 @@ jobs: - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }} - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt + - run: conda list - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - run: conda list - run: flake8 @@ -94,6 +95,7 @@ jobs: - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt - run: python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt - run: mamba install -c conda-forge git + - run: conda list - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt - run: conda list - run: flake8 From 62fec6f3a7d8a5d965906726613aa292df2baf11 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 28 May 2024 12:56:01 +0100 Subject: [PATCH 36/39] unrun GA --- .github/workflows/run-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 4720a62396..9cf1d6308b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,7 +21,6 @@ on: push: branches: - main - - python312 # run the test only if the PR is to main # turn it on if required #pull_request: From 78c8bc39faae4d76787c1ec0cc8181842e2f1426 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 28 May 2024 14:59:50 +0200 Subject: [PATCH 37/39] Remove prospector and instead install separate linters --- environment.yml | 6 +++--- setup.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 2f55afc9b5..0f68f0642d 100644 --- a/environment.yml +++ b/environment.yml @@ -68,11 +68,11 @@ dependencies: - docformatter - isort - pre-commit - - prospector >=1.9.0 - - pyroma + - pylint + - flake8 >= 7 + - pydocstyle # Not on conda forge - vprof - yamllint - yapf - pip: - - flake8 >=7 - ESMValTool_sample_data diff --git a/setup.py b/setup.py index f89385fbd2..c969fa8f67 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,8 @@ 'isort', 'flake8>=7', 'pre-commit', - # 'prospector', # pip installs old v0.12, keep conda-forge one + 'pylint', + 'pydocstyle', 'vprof', 'yamllint', 'yapf', From d6cf2deec7e7454719d115af1285314fbcf6b00d Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 28 May 2024 14:16:49 +0100 Subject: [PATCH 38/39] run ga one last time --- .github/workflows/run-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 9cf1d6308b..4720a62396 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,6 +21,7 @@ on: push: branches: - main + - python312 # run the test only if the PR is to main # turn it on if required #pull_request: From 80325cefabc7574f84055f0c13849fe05c455c9f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 28 May 2024 14:28:25 +0100 Subject: [PATCH 39/39] unrun ga one last time --- .github/workflows/run-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 4720a62396..9cf1d6308b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,7 +21,6 @@ on: push: branches: - main - - python312 # run the test only if the PR is to main # turn it on if required #pull_request: