Fixups regarding Code Coverage

Author: julien-lang

.coveragerc

@@ -8,13 +8,13 @@
 # agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
 # not expressly granted therein are reserved by Shotgun Software Inc.
 #
-# coverage configuration - used by https://coveralls.io/ integration
+# coverage configuration - used by pytest-cov and codecov.io integration
 #
 #
 
 [run]
 source=shotgun_api3
 omit=
     shotgun_api3/lib/httplib2/*
-    shotgun_api3/lib/certify/*
+    shotgun_api3/lib/certifi/*
     shotgun_api3/lib/pyparsing.py

README.md

@@ -2,7 +2,7 @@
 [![Reference Documentation](http://img.shields.io/badge/Reference-documentation-blue.svg?logo=wikibooks&logoColor=f5f5f5)](http://developer.shotgridsoftware.com/python-api)
 
 [![Build Status](https://dev.azure.com/shotgun-ecosystem/Python%20API/_apis/build/status/shotgunsoftware.python-api?branchName=master)](https://dev.azure.com/shotgun-ecosystem/Python%20API/_build/latest?definitionId=108&branchName=master)
-[![Coverage Status](https://coveralls.io/repos/github/shotgunsoftware/python-api/badge.svg?branch=master)](https://coveralls.io/github/shotgunsoftware/python-api?branch=master)
+[![Coverage Status](https://codecov.io/gh/shotgunsoftware/python-api/branch/master/graph/badge.svg)](https://codecov.io/gh/shotgunsoftware/python-api)
 
 # Flow Production Tracking Python API
 
@@ -11,7 +11,7 @@ Autodesk provides a simple Python-based API for accessing Flow Production Tracki
 The latest version can always be found at https://github.com/shotgunsoftware/python-api
 
 ## Documentation
-Tutorials and detailed documentation about the Python API are available at http://developer.shotgridsoftware.com/python-api).
+Tutorials and detailed documentation about the Python API are available at http://developer.shotgridsoftware.com/python-api.
 
 Some useful direct links:
 
@@ -31,12 +31,11 @@ You can see the [full history of the Python API on the documentation site](http:
 Integration and unit tests are provided.
 
 - All tests require:
-    - The [nose unit testing tools](http://nose.readthedocs.org),
-    - The [nose-exclude nose plugin](https://pypi.org/project/nose-exclude/)
-    - (Note: Running `pip install -r tests/ci_requirements.txt` will install this package)
+    - [pytest](https://docs.pytest.org/) and related plugins
+    - (Note: Running `pip install -r tests/requirements.txt` will install all required packages)
 - A `tests/config` file (you can copy an example from `tests/example_config`).
-- Tests can be run individually like this: `nosetests --config="nose.cfg" tests/test_client.py`
-    - Make sure to not forget the `--config="nose.cfg"` option. This option tells nose to use our config file.
-- `test_client` and `tests_unit` use mock server interaction and do not require a Flow Production Tracking instance to be available (no modifications to `tests/config` are necessary).
+- Tests can be run individually like this: `pytest tests/test_client.py`
+- To run all tests: `pytest`
+- To run tests with coverage: `pytest --cov shotgun_api3 --cov-report html`
+- `test_client` and `test_unit` use mock server interaction and do not require a Flow Production Tracking instance to be available (no modifications to `tests/config` are necessary).
 - `test_api` and `test_api_long` *do* require a Flow Production Tracking instance, with a script key available for the tests. The server and script user values must be supplied in the `tests/config` file. The tests will add test data to your server based on information in your config. This data will be manipulated by the tests, and should not be used for other purposes.
-- To run all of the tests, use the shell script `run-tests`.

azure-pipelines-templates/run-tests.yml

@@ -28,9 +28,17 @@
 
 # This is the list of parameters for this template and their default values.
 parameters:
-  os_name: ''
-  vm_image: ''
-  python_version: ''
+  - name: codecov_download_url
+    type: string
+
+  - name: os_name
+    type: string
+
+  - name: python_version
+    type: string
+
+  - name: vm_image
+    type: string
 
 jobs:
   # The job will be named after the OS and Azure will suffix the strategy to make it unique
@@ -63,7 +71,7 @@ jobs:
       script: |
         pip install --upgrade pip
         pip install --upgrade setuptools wheel
-        pip install --upgrade --requirement tests/ci_requirements.txt
+        pip install --upgrade --requirement tests/requirements.txt
 
   # The {{}} syntax is meant for the the pre-processor of Azure pipeline. Every statement inside
   # a {{}} block will be evaluated and substituted before the file is parsed to create the jobs.
@@ -79,18 +87,18 @@ jobs:
         Import-Certificate -FilePath $cert_file.FullName -CertStoreLocation Cert:\LocalMachine\Root
       displayName: Updating OS Certificates
 
-  # Runs the tests and generates test coverage. The tests results are uploaded to Azure Pipelines in the
-  # Tests tab of the build and each test run will be named after the --test-run-title argument to pytest,
-  # for example 'Windows - 2.7'
+  # Runs the tests and generates both test report and code coverage
   - task: Bash@3
     displayName: Running tests
     inputs:
       targetType: inline
       script: |
         cp ./tests/example_config ./tests/config
-        pytest --durations=0 -v \
-          --cov shotgun_api3 --cov-report xml \
-          --test-run-title="${{ parameters.os_name }}-${{ parameters.python_version }}"
+        python -m pytest \
+          --verbose \
+          --nunit-xml=test-results.xml \
+          --cov \
+          --cov-report xml \
     env:
       # Pass the values needed to authenticate with the Flow Production Tracking site and create some entities.
       # Remember, on a pull request from a client or on forked repos, those variables
@@ -124,22 +132,31 @@ jobs:
       SG_TASK_CONTENT: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }}
       SG_PLAYLIST_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }}
 
-  # Upload the code coverage result to codecov.io.
-  - ${{ if eq(parameters.os_name, 'Windows') }}:
-    - powershell: |
-        $ProgressPreference = 'SilentlyContinue'
-        Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
-        .\codecov.exe -f coverage.xml
-      displayName: Uploading code coverage
-  - ${{ elseif eq(parameters.os_name, 'Linux') }}:
-    - script: |
-        curl -Os https://uploader.codecov.io/latest/linux/codecov
-        chmod +x codecov
-        ./codecov -f coverage.xml
-      displayName: Uploading code coverage
-  - ${{ else }}:
-    - script: |
-        curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov
+  - task: PublishTestResults@2
+    displayName: Publish test results
+    inputs:
+      testResultsFormat: NUnit
+      testResultsFiles: test-results.xml
+      testRunTitle: "${{ parameters.os_name }} - Python ${{ parameters.python_version }}"
+      failTaskOnFailureToPublishResults: true
+      failTaskOnMissingResultsFile: true
+    condition: succeededOrFailed()
+
+  - task: PublishCodeCoverageResults@2
+    displayName: Publish code coverage
+    inputs:
+      summaryFileLocation: coverage.xml
+      failIfCoverageEmpty: true
+
+  - task: Bash@3
+    displayName: Uploading coverage to Codecov.io
+    inputs:
+      targetType: inline
+      script: |
+        curl -Os "${{ parameters.codecov_download_url }}"
         chmod +x codecov
-        ./codecov -f coverage.xml
-      displayName: Uploading code coverage
+        ./codecov  \
+          --file coverage.xml \
+          --flags ${{ parameters.os_name }} \
+          --flags "Python ${{ parameters.python_version }}" \
+          --name "Tests ran on ${{ parameters.os_name }} with Python ${{ parameters.python_version }}" \

azure-pipelines.yml

@@ -72,12 +72,21 @@ parameters:
     default:
       - name: Linux
         vm_image: ubuntu-latest
+        codecov_download_url: https://uploader.codecov.io/latest/linux/codecov
 
       - name: macOS
         vm_image: macOS-latest
+        codecov_download_url: https://uploader.codecov.io/v0.7.3/macos/codecov
+        # macOS & codecov note:
+        # In Nov 2024 (SG-36700), we pinned macOS to codecov v0.7.3 because the
+        # macOS-latest Azure Pipelines agent is Intel (x86_64), but Codecov
+        # started shipping arm64-only binaries.
+        # When we will change the macOs image to a arm64 arch, we can use the
+        # "latest" version again.
 
       - name: Windows
         vm_image: windows-latest
+        codecov_download_url: https://uploader.codecov.io/latest/windows/codecov.exe
 
 # This here is the list of jobs we want to run for our build.
 # Jobs run in parallel.
@@ -96,6 +105,7 @@ jobs:
   - ${{ each python_version in parameters.python_versions }}:
     - template: azure-pipelines-templates/run-tests.yml
       parameters:
-        os_name: "${{ os_version.name }}"
-        vm_image: ${{ os_version.vm_image }}
+        codecov_download_url: ${{ os_version.codecov_download_url }}
+        os_name: ${{ os_version.name }}
         python_version: ${{ python_version }}
+        vm_image: ${{ os_version.vm_image }}

run-tests

@@ -8,5 +8,6 @@
 # agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
 # not expressly granted therein are reserved by Shotgun Software Inc.
 
-[nosetests]
-exclude-dir=shotgun_api3/lib
+pytest
+pytest-cov
+pytest-nunit