From e7adf141b599a455e72ecbb05e0ca0adb81324ef Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 22:38:14 +0200 Subject: [PATCH 01/16] Update minimum version of requests dependency we require as part of install_requires in setup.py to 2.26.0 when running under Python >= 3.6. This was done to avoid potential licensing issue with transitive dependency of the requests library (chardet). NOTE: Since requests 2.26.0 dropped support for Python 3.5 which we still stupport, requests >= 2.25.1 is required when running under Python 3.5. Thanks to Jarek Potiuk - @potiuk for reporting this issue. Resolves #1594. --- CHANGES.rst | 14 ++++++++++++++ requirements-tests.txt | 3 ++- setup.py | 13 ++++++++++--- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 08c14bd62d..6232fe1b74 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,20 @@ Common - Update setup.py metadata and indicate we also support Python 3.10. +- Update minimum ``requests`` version we require as part for install_requires + in setup.py to ``2.26.0`` when using Python >= 3.6. + + This was done to avoid licensing issue with transitive dependency + (``chardet``). + + NOTE: requests ``>=2.25.1`` will be used when using Python 3.5 since 2.26.0 + doesn't support Python 3.5 anymore. + + For more context, see https://github.com/psf/requests/pull/5797. + (GITHUB-1594) + + Reported by Jarek Potiuk - @potiuk. + Storage ~~~~~~~ diff --git a/requirements-tests.txt b/requirements-tests.txt index 0be0ae3349..85ad7592e3 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,7 +5,8 @@ pylint==2.4.4 mock==3.0.5 codecov==2.1.10 coverage==4.5.4 -requests +requests>=2.25.0; python_version <= '3.5' +requests>=2.26.0; python_version >= '3.6' requests_mock # 5.3.2 is latest version which still supports Python 3.5, >= 6.2.5 is needed for Python 3.10 pytest==5.3.2; python_version <= '3.5' diff --git a/setup.py b/setup.py index c04e0f117a..0f3ef37af0 100644 --- a/setup.py +++ b/setup.py @@ -173,9 +173,16 @@ def get_data_files(dname, ignore=None, parent=None): # setuptools >= 36.2 # For installation, minimum required pip version is 1.4 # Reference: https://hynek.me/articles/conditional-python-dependencies/ -INSTALL_REQUIREMENTS = [ - 'requests>=2.5.0', -] +# We rely on >= 2.26.0 to avoid issues with LGL transitive dependecy +# See https://github.com/apache/libcloud/issues/1594 for more context +INSTALL_REQUIREMENTS = [] + +if sys.version_info < (3, 6, 0): + # requests 2.26.0 doesn't support Python 3.5 anymore + INSTALL_REQUIREMENTS.append('requests>=2.25.1') +else: + INSTALL_REQUIREMENTS.append('requests>=2.26.0') + setuptools_version = tuple(setuptools.__version__.split(".")[0:2]) setuptools_version = tuple([int(c) for c in setuptools_version]) From 4d25b241ded95fba59d3f31947a1d4757fab58a1 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:07:57 +0200 Subject: [PATCH 02/16] Run dist install + dist install wheels checks on CI for multiple Python versions. --- .github/workflows/main.yml | 48 ++++++++++++++++++++++++ scripts/dist_install_check.sh | 36 ++++++++++++++++++ tox.ini | 70 +++++++++++++++++++++-------------- 3 files changed, 127 insertions(+), 27 deletions(-) create mode 100755 scripts/dist_install_check.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7648282e5..95734405f4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,6 +84,54 @@ jobs: run: | script -e -c "tox -e py${{ matrix.python_version }}" + dist_install_checks: + name: Dist Install Checks + runs-on: ${{ matrix.os }} + + needs: pre_job + if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} + + strategy: + fail-fast: false + matrix: + python_version: + - 3.5 + - 3.6 + - 3.7 + os: + - ubuntu-latest + + steps: + - uses: actions/checkout@master + with: + fetch-depth: 1 + + - name: Use Python ${{ matrix.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + + - name: Install OS / deb dependencies + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc libvirt-dev + + - name: Cache Python Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-tests.txt', '') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python Dependencies + run: | + pip install "tox==3.24.4" + + - name: Run tox target + run: | + script -e -c "tox -e py${{ matrix.python_version }}-dist,py${{ matrix.python_version }}-dist-wheel" + code_coverage: name: Generate Code Coverage runs-on: ubuntu-latest diff --git a/scripts/dist_install_check.sh b/scripts/dist_install_check.sh new file mode 100755 index 0000000000..0d340b4408 --- /dev/null +++ b/scripts/dist_install_check.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Verify library installs without any dependencies when using python setup.py +# install +echo "Running dist checks" +pip show requests && exit 1 +pip show typing && exit 1 +pip show enum34 && exit 1 +pip show apache-libcloud + +# Install the library +python setup.py install +pip show apache-libcloud + +# Verify all dependencies were installed +pip show requests +pip show typing && exit 1 +pip show enum34 && exit 1 + +echo "Done" diff --git a/tox.ini b/tox.ini index b1d321527e..fea4724b29 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,8 @@ deps = basepython = pypypy3.5: pypy3.5 pypypy3: pypy3 - py3.5: python3.5 - py3.6: python3.6 + {py3.5,py3.5-dist,py3.5-dist-wheel}: python3.5 + {py3.6,py3.6-dist,py3.6-dist-wheel}: python3.6 {py3.7,docs,checks,lint,pylint,mypy,coverage,docs,py3.7-dist,py3.7-dist-wheel}: python3.7 {py3.8,py3.8-windows,integration-storage}: python3.8 {py3.9}: python3.9 @@ -33,7 +33,7 @@ deps = fasteners setuptools==42.0.2 -[testenv:py3.7-dist] +[testenv:py3.5-dist] # Verify library installs without any dependencies when using python setup.py # install skipdist = True @@ -43,16 +43,45 @@ recreate = True deps = # Ensure those packages are not installed. If they are, it indicates unclean # environment so those checks won't work correctly -commands = bash -c "pip show requests && exit 1 || exit 0" - bash -c "pip show typing && exit 1 || exit 0" - bash -c "pip show enum34 && exit 1 || exit 0" - bash -c "pip show apache-libcloud || exit 0" - python setup.py install - pip show apache-libcloud - # Verify all dependencies were installed - pip show requests - bash -c "pip show typing && exit 1 || exit 0" - bash -c "pip show enum34 && exit 1 || exit 0" +commands = bash ./scripts/dist_install_check.sh + +[testenv:py3.5-dist-wheel] +# Verify library installs without any dependencies when using built wheel +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_wheel_install_check.sh" + +[testenv:py3.6-dist] +# Verify library installs without any dependencies when using python setup.py +# install +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_install_check.sh" + +[testenv:py3.6-dist-wheel] +# Verify library installs without any dependencies when using built wheel +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_wheel_install_check.sh" + +[testenv:py3.7-dist] +# Verify library installs without any dependencies when using python setup.py +# install +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_install_check.sh" [testenv:py3.7-dist-wheel] # Verify library installs without any dependencies when using built wheel @@ -61,20 +90,7 @@ recreate = True # NOTE: We intentionally set empty deps to ensure it works on a clean # environment without any dependencies deps = -# Ensure those packages are not installed. If they are, it indicates unclean -# environment so those checks won't work correctly -commands = bash -c "pip show requests && exit 1 || exit 0" - bash -c "pip show typing && exit 1 || exit 0" - bash -c "pip show enum34 && exit 1 || exit 0" - bash -c "pip show apache-libcloud || exit 0" - bash -c "rm -rf dist/apache_libcloud-*.whl" - pip install wheel - python setup.py bdist_wheel - bash -c "pip install dist/apache_libcloud-*.whl" - # Verify all dependencies were installed - pip show requests - bash -c "pip show typing && exit 1 || exit 0" - bash -c "pip show enum34 && exit 1 || exit 0" +commands = bash -c "./scripts/dist_wheel_install_check.sh" [testenv:docs] deps = pyopenssl==19.1.0 From 29218a8a85f96f8822050c1a7bce433afadce6bd Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:15:53 +0200 Subject: [PATCH 03/16] Fold it into a single job to reduce number of parallel jobs and speed things up. Also run it under Python 3.8 and Python 3.9. --- .github/workflows/main.yml | 51 +++----------------------------------- tox.ini | 42 +++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95734405f4..2dbcdfa833 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,7 @@ jobs: github_token: ${{ github.token }} unit_tests: - name: Run Unit Tests + name: Run Unit Tests And Install Checks runs-on: ${{ matrix.os }} needs: pre_job @@ -80,55 +80,12 @@ jobs: run: | pip install "tox==3.24.4" - - name: Run tox target + - name: Run unit tests tox target run: | script -e -c "tox -e py${{ matrix.python_version }}" - dist_install_checks: - name: Dist Install Checks - runs-on: ${{ matrix.os }} - - needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} - - strategy: - fail-fast: false - matrix: - python_version: - - 3.5 - - 3.6 - - 3.7 - os: - - ubuntu-latest - - steps: - - uses: actions/checkout@master - with: - fetch-depth: 1 - - - name: Use Python ${{ matrix.python_version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python_version }} - - - name: Install OS / deb dependencies - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc libvirt-dev - - - name: Cache Python Dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements-tests.txt', '') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python Dependencies - run: | - pip install "tox==3.24.4" - - - name: Run tox target + - name: Run dist installchecks tox target + if: ${{ matrix.python_version != 'pypy3' }} run: | script -e -c "tox -e py${{ matrix.python_version }}-dist,py${{ matrix.python_version }}-dist-wheel" diff --git a/tox.ini b/tox.ini index fea4724b29..0610526c7b 100644 --- a/tox.ini +++ b/tox.ini @@ -15,8 +15,8 @@ basepython = {py3.5,py3.5-dist,py3.5-dist-wheel}: python3.5 {py3.6,py3.6-dist,py3.6-dist-wheel}: python3.6 {py3.7,docs,checks,lint,pylint,mypy,coverage,docs,py3.7-dist,py3.7-dist-wheel}: python3.7 - {py3.8,py3.8-windows,integration-storage}: python3.8 - {py3.9}: python3.9 + {py3.8,py3.8-windows,integration-storage,py3.8-dist,py3.8-dist-wheel}: python3.8 + {py3.9,py3.9-dist,py3.9-dist-wheel}: python3.9 {py3.10}: python3.10 setenv = CRYPTOGRAPHY_ALLOW_OPENSSL_102=1 @@ -92,6 +92,44 @@ recreate = True deps = commands = bash -c "./scripts/dist_wheel_install_check.sh" +[testenv:py3.8-dist] +# Verify library installs without any dependencies when using python setup.py +# install +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_install_check.sh" + +[testenv:py3.8-dist-wheel] +# Verify library installs without any dependencies when using built wheel +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_wheel_install_check.sh" + +[testenv:py3.9-dist] +# Verify library installs without any dependencies when using python setup.py +# install +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_install_check.sh" + +[testenv:py3.9-dist-wheel] +# Verify library installs without any dependencies when using built wheel +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_wheel_install_check.sh" + [testenv:docs] deps = pyopenssl==19.1.0 fasteners From 636dc141664816e60092b66ff6b51508ba5b09ea Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:16:36 +0200 Subject: [PATCH 04/16] Add missing file. --- scripts/dist_wheel_install_check.sh | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 scripts/dist_wheel_install_check.sh diff --git a/scripts/dist_wheel_install_check.sh b/scripts/dist_wheel_install_check.sh new file mode 100755 index 0000000000..c077e631b3 --- /dev/null +++ b/scripts/dist_wheel_install_check.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Verify library installs without any dependencies when using built wheel + +echo "Running dist wheel checks" + +# Ensure those packages are not installed. If they are, it indicates unclean +# environment so those checks won't work correctly +pip show requests && exit 1 +pip show typing && exit 1 +pip show enum34 && exit 1 +pip show apache-libcloud +rm -rf dist/apache_libcloud-*.whl + +pip install wheel +python setup.py bdist_wheel +pip install dist/apache_libcloud-*.whl + +# Verify all dependencies were installed +pip show requests +pip show typing && exit 1 +pip show enum34 && exit 1 + +echo "Done" From fb5486a9664fe5d1e3c3316e620837d12b89142c Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:19:03 +0200 Subject: [PATCH 05/16] Fix small issue detected by pylint. --- libcloud/common/openstack_identity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/common/openstack_identity.py b/libcloud/common/openstack_identity.py index 61f45e649b..307fa861ca 100644 --- a/libcloud/common/openstack_identity.py +++ b/libcloud/common/openstack_identity.py @@ -1422,7 +1422,7 @@ def __init__(self, auth_url, user_id, key, tenant_name=None, Tenant, domain and scope options are ignored as they are contained within the app credential itself and can't be changed. """ - super(OpenStackIdentity_3_0_Connection, + super(OpenStackIdentity_3_0_Connection_AppCred, self).__init__(auth_url=auth_url, user_id=user_id, key=key, From 1b66d83837f15a794f6f3ff71e2fd7627dd18168 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:22:59 +0200 Subject: [PATCH 06/16] Fix typo. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2dbcdfa833..87a51f9d38 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,7 +84,7 @@ jobs: run: | script -e -c "tox -e py${{ matrix.python_version }}" - - name: Run dist installchecks tox target + - name: Run dist install checks tox target if: ${{ matrix.python_version != 'pypy3' }} run: | script -e -c "tox -e py${{ matrix.python_version }}-dist,py${{ matrix.python_version }}-dist-wheel" From 3a69dff59df9e8e73408358d3b7dd98182c4ddd0 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:24:10 +0200 Subject: [PATCH 07/16] Also run dist checks under Python 3.10. --- tox.ini | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0610526c7b..7e66ce40b5 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ basepython = {py3.7,docs,checks,lint,pylint,mypy,coverage,docs,py3.7-dist,py3.7-dist-wheel}: python3.7 {py3.8,py3.8-windows,integration-storage,py3.8-dist,py3.8-dist-wheel}: python3.8 {py3.9,py3.9-dist,py3.9-dist-wheel}: python3.9 - {py3.10}: python3.10 + {py3.10,py3.10-dist,py3.10-dist-wheel}: python3.10 setenv = CRYPTOGRAPHY_ALLOW_OPENSSL_102=1 commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py @@ -130,6 +130,25 @@ recreate = True deps = commands = bash -c "./scripts/dist_wheel_install_check.sh" +[testenv:py3.10-dist] +# Verify library installs without any dependencies when using python setup.py +# install +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_install_check.sh" + +[testenv:py3.10-dist-wheel] +# Verify library installs without any dependencies when using built wheel +skipdist = True +recreate = True +# NOTE: We intentionally set empty deps to ensure it works on a clean +# environment without any dependencies +deps = +commands = bash -c "./scripts/dist_wheel_install_check.sh" + [testenv:docs] deps = pyopenssl==19.1.0 fasteners From 77a8d6558de92acaa0b0f2f8d3c7563089a15253 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:27:49 +0200 Subject: [PATCH 08/16] Update dist check. --- scripts/dist_install_check.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/dist_install_check.sh b/scripts/dist_install_check.sh index 0d340b4408..89807d6a0e 100755 --- a/scripts/dist_install_check.sh +++ b/scripts/dist_install_check.sh @@ -19,6 +19,11 @@ # Verify library installs without any dependencies when using python setup.py # install echo "Running dist checks" + +python --version + +# Ensure those packages are not installed. If they are, it indicates unclean +# environment so those checks won't work correctly pip show requests && exit 1 pip show typing && exit 1 pip show enum34 && exit 1 From 592abbf85f109be1c56d576c239966c791c741e6 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:34:49 +0200 Subject: [PATCH 09/16] Print python version. --- scripts/dist_install_check.sh | 3 +-- scripts/dist_wheel_install_check.sh | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/dist_install_check.sh b/scripts/dist_install_check.sh index 89807d6a0e..1850d8f828 100755 --- a/scripts/dist_install_check.sh +++ b/scripts/dist_install_check.sh @@ -18,8 +18,7 @@ # Verify library installs without any dependencies when using python setup.py # install -echo "Running dist checks" - +echo "Running dist install checks" python --version # Ensure those packages are not installed. If they are, it indicates unclean diff --git a/scripts/dist_wheel_install_check.sh b/scripts/dist_wheel_install_check.sh index c077e631b3..100d7d783a 100755 --- a/scripts/dist_wheel_install_check.sh +++ b/scripts/dist_wheel_install_check.sh @@ -18,7 +18,8 @@ # Verify library installs without any dependencies when using built wheel -echo "Running dist wheel checks" +echo "Running dist wheel install checks" +python --version # Ensure those packages are not installed. If they are, it indicates unclean # environment so those checks won't work correctly From 464c459e9d213463aaa76f74d6060a6c4e879f82 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 24 Oct 2021 23:44:36 +0200 Subject: [PATCH 10/16] Move the assignment to make race less likely to occur. --- libcloud/test/dns/test_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libcloud/test/dns/test_base.py b/libcloud/test/dns/test_base.py index bb699f4730..efdfefc8d6 100644 --- a/libcloud/test/dns/test_base.py +++ b/libcloud/test/dns/test_base.py @@ -83,6 +83,8 @@ def test_export_zone_to_bind_format_success(self): self.driver.list_records = Mock() self.driver.list_records.return_value = mock_records + now = datetime.datetime.utcnow() + result = self.driver.export_zone_to_bind_format(zone=zone) self.driver.export_zone_to_bind_zone_file(zone=zone, file_path=self.tmp_path) @@ -93,7 +95,6 @@ def test_export_zone_to_bind_format_success(self): lines1 = result.split('\n') lines2 = content.split('\n') - now = datetime.datetime.utcnow() date_str = "%s-%s-%s %s:%s:%s" % (now.year, zero_pad(now.month), zero_pad(now.day), zero_pad(now.hour), From 71f4f1ff6eee19a742424954c3f0764a0a8b3df3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 26 Oct 2021 14:38:07 +0200 Subject: [PATCH 11/16] Add a new CI job which runs on daily basis and verifies Libcloud can be installed from pip using all the supported versions. --- .github/workflows/install_test.yml | 47 ++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/install_test.yml diff --git a/.github/workflows/install_test.yml b/.github/workflows/install_test.yml new file mode 100644 index 0000000000..39994b9db3 --- /dev/null +++ b/.github/workflows/install_test.yml @@ -0,0 +1,47 @@ +# Workflow which verifies that the latest stable version can be installed from +# pip on all the supported Python versions +name: Install stable version using pip + +on: + push: + branches: + - requests_changes + pull_request: + branches: + - requests_changes + schedule: + - cron: '0 13 * * *' + - cron: '0 2 * * *' + +jobs: + install_and_verify: + name: Install latest stable version + runs-on: ubuntu-latest + timeout-minutes: 5 + + strategy: + fail-fast: false + matrix: + python_version: + - 3.5 + - 3.6 + - 3.7 + - 3.8 + - 3.9 + - pypy3 + + steps: + - uses: actions/checkout@master + with: + fetch-depth: 1 + + - name: Use Python ${{ matrix.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + + - name: Install Libcloud + run: | + pip show apache-libcloud && exit 1 + pip install apache-libcloud + pip show apache-libcloud From 6a5eafcd0328d4a874ba676196b0da116fc76dd7 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 26 Oct 2021 14:46:05 +0200 Subject: [PATCH 12/16] Add skip duplicates pre job to CodeQL workflow. --- .github/workflows/codeql-analysis.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c6dfc5fb5e..929d8ed6f3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -9,10 +9,34 @@ on: - cron: '0 3 * * *' jobs: + # Special job which skips duplicate jobs + pre_job: + name: Skip Duplicate Jobs Pre Job + runs-on: ubuntu-latest + # Map a step output to a job output + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - name: Checkout code + uses: actions/checkout@master + with: + persist-credentials: false + submodules: recursive + + - id: skip_check + # NOTE: We store action as submodule since ASF doesn't allow directly referencing external + # actions + uses: ./.github/actions/skip-duplicate-actions # v3.4.0 + with: + github_token: ${{ github.token }} + analyze: name: Analyze runs-on: ubuntu-latest + needs: pre_job + if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} + strategy: fail-fast: false matrix: From eb655e4e0e5a6a9f551517f344e729a337cb757b Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 26 Oct 2021 14:47:56 +0200 Subject: [PATCH 13/16] Add job timeouts to all the jobs. --- .github/workflows/codeql-analysis.yml | 1 + .github/workflows/install_test.yml | 3 ++- .github/workflows/main.yml | 1 + .github/workflows/publish_pricing_to_s3.yml | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 929d8ed6f3..16cd806543 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,6 +33,7 @@ jobs: analyze: name: Analyze runs-on: ubuntu-latest + timeout-minutes: 12 needs: pre_job if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} diff --git a/.github/workflows/install_test.yml b/.github/workflows/install_test.yml index 39994b9db3..a67f741d39 100644 --- a/.github/workflows/install_test.yml +++ b/.github/workflows/install_test.yml @@ -17,7 +17,7 @@ jobs: install_and_verify: name: Install latest stable version runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 2 strategy: fail-fast: false @@ -42,6 +42,7 @@ jobs: - name: Install Libcloud run: | + python --version pip show apache-libcloud && exit 1 pip install apache-libcloud pip show apache-libcloud diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 87a51f9d38..e02f9f033f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,6 +35,7 @@ jobs: unit_tests: name: Run Unit Tests And Install Checks runs-on: ${{ matrix.os }} + timeout-minutes: 8 needs: pre_job if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} diff --git a/.github/workflows/publish_pricing_to_s3.yml b/.github/workflows/publish_pricing_to_s3.yml index 1f34e8f90c..48ba4c21c3 100644 --- a/.github/workflows/publish_pricing_to_s3.yml +++ b/.github/workflows/publish_pricing_to_s3.yml @@ -10,6 +10,7 @@ jobs: generate_and_publish_pricing_data: name: Generate and Publish Pricing file to S3 runs-on: ubuntu-latest + timeout-minutes: 5 strategy: matrix: From 819d2857b4c4092eaf346ee15bf722611e802700 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 26 Oct 2021 14:50:25 +0200 Subject: [PATCH 14/16] Remove testing changes. --- .github/workflows/install_test.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/install_test.yml b/.github/workflows/install_test.yml index a67f741d39..da7e460131 100644 --- a/.github/workflows/install_test.yml +++ b/.github/workflows/install_test.yml @@ -3,12 +3,6 @@ name: Install stable version using pip on: - push: - branches: - - requests_changes - pull_request: - branches: - - requests_changes schedule: - cron: '0 13 * * *' - cron: '0 2 * * *' From 7f61280d9340d984e3cb19b0a74b0a06470bf5e0 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 26 Oct 2021 14:53:58 +0200 Subject: [PATCH 15/16] Try pinning docutils to see if it fixes new build failure which just started happening today. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 7e66ce40b5..55dfe9c9d3 100644 --- a/tox.ini +++ b/tox.ini @@ -170,6 +170,8 @@ deps = pyopenssl==19.1.0 fasteners rstcheck requests_mock + # Pinned due to a bug in a newer version + docutils<=0.16 changedir = docs commands = pip install sphinx==3.3.1 rstcheck --report warning ../README.rst From 7ed2b02d6b34147225dc4bd7c0632857d1daef82 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 26 Oct 2021 14:55:13 +0200 Subject: [PATCH 16/16] Re-regenerate supported providers tables. --- docs/compute/_supported_methods_block_storage.rst | 2 +- docs/compute/_supported_methods_key_pair_management.rst | 2 +- docs/compute/_supported_methods_main.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/compute/_supported_methods_block_storage.rst b/docs/compute/_supported_methods_block_storage.rst index ec5e127f62..6926eabfe4 100644 --- a/docs/compute/_supported_methods_block_storage.rst +++ b/docs/compute/_supported_methods_block_storage.rst @@ -12,7 +12,7 @@ Provider list volumes create volume destroy volume `Brightbox`_ no no no no no no no `BSNL`_ no no no no no no no `Cloudscale`_ no no no no no no no -`CloudSigma (API v2.0)`_ no no no no no no no +`CloudSigma (API v2.0)`_ yes yes yes yes yes no no `CloudStack`_ yes yes yes yes yes no yes `Cloudwatt`_ yes yes yes yes yes yes yes `DigitalOcean`_ yes yes yes yes yes yes yes diff --git a/docs/compute/_supported_methods_key_pair_management.rst b/docs/compute/_supported_methods_key_pair_management.rst index 756c80483a..f3f17fe6b3 100644 --- a/docs/compute/_supported_methods_key_pair_management.rst +++ b/docs/compute/_supported_methods_key_pair_management.rst @@ -12,7 +12,7 @@ Provider list key pairs get key pair create key pai `Brightbox`_ no no no no no no `BSNL`_ no no no no no no `Cloudscale`_ no no no no no no -`CloudSigma (API v2.0)`_ no no no no no no +`CloudSigma (API v2.0)`_ yes yes yes yes no yes `CloudStack`_ yes yes yes yes no yes `Cloudwatt`_ yes yes yes yes no yes `DigitalOcean`_ yes yes yes no no yes diff --git a/docs/compute/_supported_methods_main.rst b/docs/compute/_supported_methods_main.rst index 379506cb3d..144928f991 100644 --- a/docs/compute/_supported_methods_main.rst +++ b/docs/compute/_supported_methods_main.rst @@ -12,7 +12,7 @@ Provider list nodes create node reboot node destroy `Brightbox`_ yes yes no yes no no yes yes no `BSNL`_ yes yes yes yes yes yes yes yes yes `Cloudscale`_ yes yes yes yes yes yes yes yes no -`CloudSigma (API v2.0)`_ yes yes no yes yes yes yes yes no +`CloudSigma (API v2.0)`_ yes yes yes yes yes yes yes yes no `CloudStack`_ yes yes yes yes no no yes yes yes `Cloudwatt`_ yes yes yes yes yes yes yes yes yes `DigitalOcean`_ yes yes yes yes no no yes yes no