From 45da0a0b27f5eb0987a741c3e3f6781502b7e5a7 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Mon, 21 Feb 2022 12:28:39 +0100 Subject: [PATCH 1/4] add tox + remove coverage from pytest default config By making testing and coverage separate parts of the configuration, VSCode no longer chokes on discovering tests. This should also make test discovery easier in IDEs like PyCharm. The main problem was that coverage was being treated as default part of testing by always passing coverage flags to pytest (this was also noted in a comment in setup.cfg). Moving this to pyproject.toml may have been unnecessary in the end, but the resources I found online suggested this was the way to go. Adding tox provides a solution to #38. --- tests/test_project.py | 9 +++++++++ {{cookiecutter.directory_name}}/pyproject.toml | 18 ++++++++++++++++++ {{cookiecutter.directory_name}}/setup.cfg | 12 ++---------- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index 34642647..565d9e4d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -69,6 +69,15 @@ def test_pytest(baked_with_development_dependencies, project_env_bin_dir): assert (project_dir / 'htmlcov' / 'index.html').exists() +def test_tox(baked_with_development_dependencies, project_env_bin_dir): + project_dir = baked_with_development_dependencies + bin_dir = project_env_bin_dir + result = run([f'{bin_dir}tox'], project_dir) + assert result.returncode == 0 + assert '== 3 passed in' in result.stdout + assert (project_dir / '.tox' / 'dist' / 'my_python_package-0.1.0.zip').exists() + + def test_subpackage(baked_with_development_dependencies, project_env_bin_dir): """Test if subpackages end up in sdist and bdist_wheel distributions""" project_dir = baked_with_development_dependencies diff --git a/{{cookiecutter.directory_name}}/pyproject.toml b/{{cookiecutter.directory_name}}/pyproject.toml index 9787c3bd..3a71fd27 100644 --- a/{{cookiecutter.directory_name}}/pyproject.toml +++ b/{{cookiecutter.directory_name}}/pyproject.toml @@ -1,3 +1,21 @@ [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.coverage.run] +branch = true +source = ["{{ cookiecutter.package_name }}"] +command_line = "-m pytest" + +[tool.tox] +legacy_tox_ini = """ +[tox] +envlist = py37,py38,py39 +skip_missing_interpreters = true +[testenv] +commands = pytest +extras = dev +""" diff --git a/{{cookiecutter.directory_name}}/setup.cfg b/{{cookiecutter.directory_name}}/setup.cfg index ad06d7ae..6572b8ec 100644 --- a/{{cookiecutter.directory_name}}/setup.cfg +++ b/{{cookiecutter.directory_name}}/setup.cfg @@ -48,6 +48,7 @@ install_requires = [options.extras_require] dev = bump2version + coverage [toml] prospector[with_pyroma] isort pytest @@ -55,6 +56,7 @@ dev = sphinx sphinx_rtd_theme sphinx-autoapi + tox publishing = twine wheel @@ -62,10 +64,6 @@ publishing = [options.packages.find] include = {{ cookiecutter.package_name }}, {{ cookiecutter.package_name }}.* -[coverage:run] -branch = True -source = {{ cookiecutter.package_name }} - [isort] lines_after_imports = 2 force_single_line = 1 @@ -73,9 +71,3 @@ no_lines_before = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER known_first_party = {{ cookiecutter.package_name }} src_paths = {{ cookiecutter.package_name }},tests line_length = 120 - -[tool:pytest] -testpaths = tests -# Note that visual debugger in some editors like pycharm gets confused by coverage calculation. -# As a workaround, configure the test configuration in pycharm et al with a --no-cov argument -addopts = --cov --cov-report xml --cov-report term --cov-report html From 5d913c55f7b43520c830d70571377be682327e02 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Tue, 22 Feb 2022 11:20:33 +0100 Subject: [PATCH 2/4] test coverage separately from pytest --- setup.cfg | 1 + tests/test_project.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1e627666..8a8e9120 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ install_requires = [options.extras_require] dev = + coverage [toml] pytest<5.0.0,>=3.3.0 pytest-cookies diff --git a/tests/test_project.py b/tests/test_project.py index 565d9e4d..d99e7f23 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -65,8 +65,15 @@ def test_pytest(baked_with_development_dependencies, project_env_bin_dir): result = run([f'{bin_dir}pytest'], project_dir) assert result.returncode == 0 assert '== 3 passed in' in result.stdout - assert (project_dir / 'coverage.xml').exists() - assert (project_dir / 'htmlcov' / 'index.html').exists() + + +def test_coverage(baked_with_development_dependencies, project_env_bin_dir): + project_dir = baked_with_development_dependencies + bin_dir = project_env_bin_dir + result = run([f'{bin_dir}coverage', 'run', '-m', 'pytest'], project_dir) + assert result.returncode == 0 + assert '== 3 passed in' in result.stdout + assert (project_dir / '.coverage').exists() def test_tox(baked_with_development_dependencies, project_env_bin_dir): From a1fbec9ce9262bc1fdd8208f87cffff990006007 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Tue, 22 Feb 2022 11:49:06 +0100 Subject: [PATCH 3/4] update pytest introduction URL --- {{cookiecutter.directory_name}}/project_setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.directory_name}}/project_setup.md b/{{cookiecutter.directory_name}}/project_setup.md index c99b85ba..4c174883 100644 --- a/{{cookiecutter.directory_name}}/project_setup.md +++ b/{{cookiecutter.directory_name}}/project_setup.md @@ -42,7 +42,7 @@ help you decide which tool to use for packaging. - The `tests` folder contains: - Example tests that you should replace with your own meaningful tests (file: `test_my_module.py`) - The testing framework used is [PyTest](https://pytest.org) - - [PyTest introduction](http://pythontesting.net/framework/pytest/pytest-introduction/) + - [PyTest introduction](https://pythontest.com/pytest-book/) - PyTest is listed as a development dependency - This is configured in `setup.cfg` - The project uses [GitHub action workflows](https://docs.github.com/en/actions) to automatically run tests on GitHub infrastructure against multiple Python versions From 6d78d07e069e1c031889aa08c17e93fd70ae3100 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 24 Feb 2022 14:25:38 +0100 Subject: [PATCH 4/4] document tox and coverage in README.dev.md --- {{cookiecutter.directory_name}}/README.dev.md | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/{{cookiecutter.directory_name}}/README.dev.md b/{{cookiecutter.directory_name}}/README.dev.md index 5bbea3ef..77b17497 100644 --- a/{{cookiecutter.directory_name}}/README.dev.md +++ b/{{cookiecutter.directory_name}}/README.dev.md @@ -25,12 +25,41 @@ Afterwards check that the install directory is present in the `PATH` environment ## Running the tests -Running the tests requires an activated virtual environment with the development tools installed. +There are two ways to run tests. + +The first way requires an activated virtual environment with the development tools installed: ```shell pytest -v ``` +The second is to use `tox`, which can be installed separately (e.g. with `pip install tox`), i.e. not necessarily inside the virtual environment you use for installing `{{ cookiecutter.package_name }}`, but then builds the necessary virtual environments itself by simply running: + +```shell +tox +``` + +Testing with `tox` allows for keeping the testing environment separate from your development environment. +The development environment will typically accumulate (old) packages during development that interfere with testing; this problem is avoided by testing with `tox`. + +### Test coverage + +In addition to just running the tests to see if they pass, they can be used for coverage statistics, i.e. to determine how much of the package's code is actually executed during tests. +In an activated virtual environment with the development tools installed, inside the package directory, run: + +```shell +coverage run +``` + +This runs tests and stores the result in a `.coverage` file. +To see the results on the command line, run + +```shell +coverage report +``` + +`coverage` can also generate output in HTML and other formats; see `coverage help` for more information. + ## Running linters locally For linting we will use [prospector](https://pypi.org/project/prospector/) and to sort imports we will use