pre-commit hook + ruff usage (#2013)

Author: cpelley

.github/workflows/ci.yml

@@ -11,13 +11,15 @@ jobs:
       fail-fast: false
       matrix:
         env: [environment_a, environment_b, conda-forge]
+
     steps:
     - uses: actions/checkout@v4
     - uses: actions/cache@v4
       id: cache
       with:
         path: /usr/share/miniconda/envs/im${{ matrix.env }}
         key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
@@ -26,29 +28,34 @@ jobs:
         conda install -q -n base -c conda-forge -c nodefaults mamba
         rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488
         mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         conda info
         conda list
+
     - name: sphinx documentation
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         make -C doc html SPHINXOPTS="-W --keep-going"
+
     - name: pytest without coverage
       if: matrix.env == 'conda-forge'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         pytest
+
     - name: pytest with coverage
       if: matrix.env != 'conda-forge'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         pytest --cov=improver --cov-report xml:coverage.xml
+
     - name: codacy upload
       if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
       run: |
@@ -57,76 +64,81 @@ jobs:
         python-codacy-coverage -v -r coverage.xml
       env:
         CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
+
     - name: codecov upload
       uses: codecov/codecov-action@v4
       with:
         name:  ${{ matrix.env }}
       if: matrix.env != 'conda_forge'
-  Codestyle-and-flake8:
+
+  pre-commit-checks:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
-      matrix:
-        env: [environment_a]
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/cache@v4
-      id: cache
+
+    - name: improver checkout
+      uses: actions/checkout@v4
       with:
-        path: /usr/share/miniconda/envs/im${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
-    - name: conda env update
-      if: steps.cache.outputs.cache-hit != 'true'
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda update -q -n base -c defaults conda
-        conda install -q -n base -c conda-forge -c nodefaults mamba
-        rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488
-        mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
-    - name: conda info
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate im${{ matrix.env }}
-        conda info
-        conda list
-    - name: isort
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.7
+        cache: 'pip'
+
+    - name: pip install pre-commit
       run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate im${{ matrix.env }}
-        isort --check-only .
-    - name: black
+        pip install pre-commit
+
+    - name: Python interpreter version sha (PYSHA)
+      run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
+
+    - name: Cache pre-commit
+      uses: actions/cache@v3
+      id: pre-commit-cache
+      with:
+        path: ~/.cache/pre-commit
+        key: pre-commit|${{ env.PYSHA }}|${{ hashFiles('.pre-commit-config.yaml') }}
+
+    - name: pre-commit install
       run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate im${{ matrix.env }}
-        black --check .
-    - name: flake8
+        pre-commit install
+
+    - name: pre-commit run
       run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate im${{ matrix.env }}
-        flake8 improver improver_tests
+        pre-commit run --verbose --color=always --all-files
+
   PR-standards:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
       - name: Check CONTRIBUTING.md
         uses: cylc/release-actions/check-shortlog@v1
+
   Safety-Bandit:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
         env: [environment_a, environment_b, conda-forge]
     steps:
+
     - uses: actions/checkout@v4
+
     - uses: actions/cache@v4
       id: cache
       with:
         path: /usr/share/miniconda/envs/im${{ matrix.env }}
         key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
@@ -135,17 +147,20 @@ jobs:
         conda install -q -n base -c conda-forge mamba
         rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488
         mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         conda info
         conda list
+
     - name: safety
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         safety check || true
+
     - name: bandit
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.4.8
+  hooks:
+    # Run the linter (must come before any formatter, black/ruff when --fix).
+    - id: ruff
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format
+- repo: local
+  hooks:
+    - id: copyright_check
+      name: Check copyright header
+      types: [python]
+      args: [ --fix ]
+      language: system
+      entry: ./copyright_check
+    - id: init_check
+      name: Check for missing __init__.py files
+      types: [python]
+      language: system
+      entry: ./init_check
+      files: ^(improver/|improver_tests/)

README.md

@@ -42,3 +42,10 @@ activate your new improver environment
 ```
 conda activate improver
 ```
+
+## Pre-commit Hook
+OPTIONAL: A pre-commit hook can be added to facilitate the development of this code base.
+Ensure that you have python available on the path, then install the pre-commit hook by running `pre-commit install` from within your working copy.
+pre-commit checks will run against modified files when you commit from then on.
+
+These pre-commit hooks will run as part of continuous integration to maintain code quality standards in the project.

bin/improver

@@ -1,7 +1,7 @@
 #!/bin/bash
-# (C) Crown copyright, Met Office. All rights reserved.
+# (C) Crown Copyright, Met Office. All rights reserved.
 #
-# This file is part of IMPROVER and is released under a BSD 3-Clause license.
+# This file is part of 'IMPROVER' and is released under the BSD 3-Clause license.
 # See LICENSE in the root of the repository for full licensing details.
 #------------------------------------------------------------------------------
 # NAME

bin/improver-tests

@@ -1,7 +1,7 @@
 #!/bin/bash
-# (C) Crown copyright, Met Office. All rights reserved.
+# (C) Crown Copyright, Met Office. All rights reserved.
 #
-# This file is part of IMPROVER and is released under a BSD 3-Clause license.
+# This file is part of 'IMPROVER' and is released under the BSD 3-Clause license.
 # See LICENSE in the root of the repository for full licensing details.
 #------------------------------------------------------------------------------
 # NAME
@@ -31,19 +31,9 @@ function get_python_files {
                                                ! -name '*~' \)`
 }
 
-function improver_test_black {
-    ${BLACK:-black} $FILES_TO_TEST
-    echo_ok "black"
-}
-
-function improver_test_isort {
-    ${ISORT:-isort} $FILES_TO_TEST
-    echo_ok "isort"
-}
-
-function improver_test_flake8 {
-    ${FLAKE8:-flake8} $FILES_TO_TEST
-    echo_ok "flake8"
+function improver_test_pre_commit {
+    pre-commit run --all-files
+    echo_ok "pre-commit"
 }
 
 function improver_test_doc {
@@ -105,19 +95,19 @@ function print_usage {
     cat <<'__USAGE__'
 improver tests [OPTIONS] [SUBTEST...] [SUBCLI...]
 
-Run black, isort, flake8, documentation, unit and CLI acceptance tests.
+Run pre-commit, documentation, unit and CLI acceptance tests.
 
 Optional arguments:
     --debug         Run in verbose mode (may take longer for CLI)
     -h, --help      Show this message and exit
 
 Arguments:
     SUBTEST         Name(s) of a subtest to run without running the rest.
-                    Valid names are: black, isort, flake8, doc, unit, cli,
-                    and recreate_checksums. The default tests are black,
-                    isort, flake8, doc, unit, and cli.  Using
-                    recreate_checksums will regenerate the test data
-                    checksum file which is used as part of the cli tests.
+                    Valid names are: pre_commit, doc, unit, cli,
+                    and recreate_checksums. The default tests are pre_commit,
+                    doc, unit, and cli.  Using recreate_checksums will
+                    regenerate the test data checksum file which is used as
+                    part of the cli tests.
 __USAGE__
 }
 
@@ -158,7 +148,7 @@ for arg in "$@"; do
         print_usage
         exit 0
         ;;
-        black|isort|flake8|doc|unit|cli|recreate_checksums)
+        pre_commit|doc|unit|cli|recreate_checksums)
         SUBTESTS="$SUBTESTS $arg"
         ;;
         $cli_tasks)
@@ -176,7 +166,7 @@ if [[ -n "$SUBTESTS" ]]; then
     TESTS="$SUBTESTS"
 else
     # Default tests.
-    TESTS="isort black flake8 doc unit cli"
+    TESTS="pre_commit doc unit cli"
 fi
 
 # If cli sub test is not specified by user, do all cli tests.