Remove top level lib directory and move tests to top level (#1108)

* Move tests to top level

* Move improver to top level

* Update pytest test discovery path

* Bulk rewrite test imports

* Move tolerance constants to improver

This avoids improver code importing from tests.

* Convert test imports to relative

* Sort all imports

Settings used: isort --recursive --atomic -w 79 -m 4 -y

* Update bin scripts to remove lib directory

* Remove unused import

* Update paths in github actions

* Fix long line lengths

Rename some duplicated directory/file names to make names shorter

* Update docs makefile

* Fix import in metadata CLI

* Rename tests to improver_tests

* Update paths for change from tests to improver_tests

Author: tjtg

.github/workflows/tests.yml

@@ -16,13 +16,13 @@ jobs:
         $CONDA/bin/conda list --export
 
     - name: Pycodestyle (PEP8 conventions)
-      run:  $CONDA/bin/pycodestyle -v lib
+      run:  $CONDA/bin/pycodestyle -v improver improver_tests
 
     - name: Sphinx-build (Doc test)
       run: cd doc; make SPHINXBUILD=$CONDA/bin/sphinx-build html
-      
+
     - name: Pytest (Unit tests)
       run:  $CONDA/bin/pytest -m "not acc"
 
     - name: Pylint (More linting goodness)
-      run: $CONDA/bin/pylint -j 0 -E --rcfile=etc/pylintrc lib
+      run: $CONDA/bin/pylint -j 0 -E --rcfile=etc/pylintrc improver improver_tests

bin/improver

@@ -58,6 +58,6 @@ fi
 
 # Put our library and scripts in the paths.
 export PATH="$IMPROVER_DIR/bin/:$PATH"
-export PYTHONPATH="$IMPROVER_DIR/lib/:${PYTHONPATH:-}"
+export PYTHONPATH="$IMPROVER_DIR/:${PYTHONPATH:-}"
 
 exec python3 -m improver.cli "$@"

bin/improver-tests

@@ -126,12 +126,12 @@ function improver_test_pycodestyle {
 
 function improver_test_pylint {
     # Pylint score generation.
-    ${PYLINT:-pylint} --rcfile=../etc/pylintrc $FILES_TO_TEST
+    ${PYLINT:-pylint} --rcfile=etc/pylintrc $FILES_TO_TEST
 }
 
 function improver_test_pylintE {
     # Pylint obvious-errors-only testing.
-    ${PYLINT:-pylint} -E --rcfile=../etc/pylintrc $FILES_TO_TEST
+    ${PYLINT:-pylint} -E --rcfile=etc/pylintrc $FILES_TO_TEST
     echo_ok "pylint -E"
 }
 
@@ -214,11 +214,10 @@ if [[ -f "${IMPROVER_SITE_INIT:=$IMPROVER_DIR/etc/site-init}" ]]; then
     . "$IMPROVER_SITE_INIT"
 fi
 
-export PYTHONPATH="$IMPROVER_DIR/lib:${PYTHONPATH:-}"
+export PYTHONPATH="$IMPROVER_DIR/:${PYTHONPATH:-}"
 
 TEST_INPUT_PWD=$(cd $PWD && pwd -P)
-cd $IMPROVER_DIR/lib
-#cd $IMPROVER_DIR
+cd $IMPROVER_DIR/
 
 # Find cli test options and format to work with case statement
 shopt -s extglob
@@ -268,7 +267,7 @@ fi
 
 # If cli sub test is not specified by user, do all cli tests.
 # Otherwise set CLISUBTEST to the sub test to run.
-CLISUBTEST="$IMPROVER_DIR/tests/"
+CLISUBTEST="$IMPROVER_DIR/improver_tests/"
 STRIPPED_TEST="$(echo -e "${TESTS}" | tr -d '[:space:]')"
 if [[ $STRIPPED_TEST == "cli" ]]; then
     if [[ -n "$SUBCLI" ]]; then

doc/Makefile

@@ -32,7 +32,7 @@ clean:
 
 .PHONY: apidoc
 apidoc:
-	sphinx-apidoc -e -P -f -o source ../lib/improver ../lib/improver/tests
+	sphinx-apidoc -e -P -f -o source ../improver ../improver_tests
 
 .PHONY: html
 html:

doc/source/conf.py

@@ -52,7 +52,7 @@
 
 SOURCE_DIR = os.path.abspath(os.path.join(
     os.path.dirname(os.path.abspath(__file__)),
-    '..', '..', 'lib'
+    '..', '..'
     ))
 
 sys.path.insert(0, SOURCE_DIR)
@@ -416,7 +416,7 @@ def run_apidoc(_):
     from sphinx.ext.apidoc import main
 
     output_dir = os.path.dirname(os.path.abspath(__file__))
-    exclude_dir = os.path.join(SOURCE_DIR, 'improver', 'tests')
+    exclude_dir = os.path.join(SOURCE_DIR, 'improver_tests')
     main(['-e', '-P', '-f', '-o', output_dir, SOURCE_DIR, exclude_dir])
 
 

lib/improver/__init__.py improver/__init__.pylib/improver/__init__.py renamed to improver/__init__.py

@@ -33,7 +33,6 @@
 
 import iris
 import numpy as np
-
 from iris.exceptions import CoordinateNotFoundError
 
 from improver import BasePlugin

lib/improver/between_thresholds.py improver/between_thresholds.pylib/improver/between_thresholds.py renamed to improver/between_thresholds.py

@@ -31,13 +31,12 @@
 """Plugin to calculate blend weights and blend data across a dimension"""
 
 from improver import BasePlugin
-from improver.blending.spatial_weights import (
-    SpatiallyVaryingWeightsFromMask)
+from improver.blending.spatial_weights import SpatiallyVaryingWeightsFromMask
 from improver.blending.weighted_blend import (
     MergeCubesForWeightedBlending, WeightedBlendAcrossWholeDimension)
 from improver.blending.weights import (
-    ChooseWeightsLinear, ChooseDefaultWeightsLinear,
-    ChooseDefaultWeightsNonLinear)
+    ChooseDefaultWeightsLinear, ChooseDefaultWeightsNonLinear,
+    ChooseWeightsLinear)
 from improver.metadata.amend import amend_attributes
 from improver.utilities.spatial import (
     check_if_grid_is_equal_area, convert_distance_into_number_of_grid_cells)