From 85b133d70e160fb6004c0408c33c8b4f98397016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Sun, 18 Sep 2022 21:21:24 +0100 Subject: [PATCH 1/6] .github: Run tests in fedora 37 as well And remove soon to be deprecated fedora 35 --- .github/common.env | 2 +- .github/compose/ci.docker-compose.yml | 10 +++++----- .github/run-ci.sh | 2 +- .github/workflows/ci.yml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/common.env b/.github/common.env index e8f0776f4..1b19378f9 100644 --- a/.github/common.env +++ b/.github/common.env @@ -1,5 +1,5 @@ # Shared common variables -CI_IMAGE_VERSION=master-533491591 +CI_IMAGE_VERSION=master-643533272 CI_TOXENV_MAIN=py36-nocover,py37-nocover,py38-nocover,py39-nocover,py310-nocover CI_TOXENV_ALL="${CI_TOXENV_MAIN}" diff --git a/.github/compose/ci.docker-compose.yml b/.github/compose/ci.docker-compose.yml index 05a704b67..0f693e6eb 100644 --- a/.github/compose/ci.docker-compose.yml +++ b/.github/compose/ci.docker-compose.yml @@ -1,7 +1,7 @@ version: '3.4' x-tests-template: &tests-template - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:35-${CI_IMAGE_VERSION:-latest} + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:36-${CI_IMAGE_VERSION:-latest} command: tox -vvvvv -- --color=yes --integration environment: TOXENV: ${CI_TOXENV_ALL} @@ -22,14 +22,14 @@ x-tests-template: &tests-template services: - fedora-35: - <<: *tests-template - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:35-${CI_IMAGE_VERSION:-latest} - fedora-36: <<: *tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:36-${CI_IMAGE_VERSION:-latest} + fedora-37: + <<: *tests-template + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:37-${CI_IMAGE_VERSION:-latest} + debian-10: <<: *tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:10-${CI_IMAGE_VERSION:-latest} diff --git a/.github/run-ci.sh b/.github/run-ci.sh index e923b9d2a..b19f8ad32 100755 --- a/.github/run-ci.sh +++ b/.github/run-ci.sh @@ -67,8 +67,8 @@ set -e if [ -z "${test_names}" ]; then runTest "lint" runTest "debian-10" - runTest "fedora-35" runTest "fedora-36" + runTest "fedora-37" else for test_name in "${test_names}"; do runTest "${test_name}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79ea923b6..c6118d30f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,8 @@ jobs: # "../compose/ci.docker-compose.yml" test-name: - debian-10 - - fedora-35 - fedora-36 + - fedora-37 - lint steps: From 95257a300479ff92718d707a0c861f197ea1cbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Sun, 18 Sep 2022 23:04:06 +0100 Subject: [PATCH 2/6] Enable python 3.11 test This is the python version in fedora 37 --- .github/common.env | 2 +- buildstream/plugins/sources/pip.py | 1 + tox.ini | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/common.env b/.github/common.env index 1b19378f9..f5db87b93 100644 --- a/.github/common.env +++ b/.github/common.env @@ -1,5 +1,5 @@ # Shared common variables CI_IMAGE_VERSION=master-643533272 -CI_TOXENV_MAIN=py36-nocover,py37-nocover,py38-nocover,py39-nocover,py310-nocover +CI_TOXENV_MAIN=py36-nocover,py37-nocover,py38-nocover,py39-nocover,py310-nocover,py311-nocover CI_TOXENV_ALL="${CI_TOXENV_MAIN}" diff --git a/buildstream/plugins/sources/pip.py b/buildstream/plugins/sources/pip.py index 6ea8476e0..381f24a72 100644 --- a/buildstream/plugins/sources/pip.py +++ b/buildstream/plugins/sources/pip.py @@ -93,6 +93,7 @@ 'python3.8', 'python3.9', 'python3.10', + 'python3.11', ] # List of allowed extensions taken from diff --git a/tox.ini b/tox.ini index a5d14a209..f388ce4a5 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ # Tox global configuration # [tox] -envlist = py36-nocover,py37-nocover,py38-nocover,py39-nocover,py310-nocover +envlist = py36-nocover,py37-nocover,py38-nocover,py39-nocover,py310-nocover,py311-nocover skip_missing_interpreters = true # @@ -13,16 +13,16 @@ skip_missing_interpreters = true [testenv] commands = # Running with coverage reporting enabled - py{36,37,38,39,310}-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs} - py{36,37,38,39,310}-!nocover: mkdir -p .coverage-reports - py{36,37,38,39,310}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} + py{36,37,38,39,310,311}-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs} + py{36,37,38,39,310,311}-!nocover: mkdir -p .coverage-reports + py{36,37,38,39,310,311}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} # Running with coverage reporting disabled - py{36,37,38,39,310}-nocover: pytest --basetemp {envtmpdir} {posargs} + py{36,37,38,39,310,311}-nocover: pytest --basetemp {envtmpdir} {posargs} deps = - py{36,37,38,39,310}: -rrequirements/requirements.txt - py{36,37,38,39,310}: -rrequirements/dev-requirements.txt - py{36,37,38,39,310}: -rrequirements/plugin-requirements.txt + py{36,37,38,39,310,311}: -rrequirements/requirements.txt + py{36,37,38,39,310,311}: -rrequirements/dev-requirements.txt + py{36,37,38,39,310,311}: -rrequirements/plugin-requirements.txt # Only require coverage and pytest-cov when using it !nocover: -rrequirements/cov-requirements.txt @@ -35,9 +35,9 @@ passenv = # These keys are not inherited by any other sections # setenv = - py{36,37,38,39,310}: COVERAGE_FILE = {envtmpdir}/.coverage + py{36,37,38,39,310,311}: COVERAGE_FILE = {envtmpdir}/.coverage whitelist_externals = - py{36,37,38,39,310}: + py{36,37,38,39,310,311}: mv mkdir From 314c303faff165735c6e27de2d9ff0ea37acde4a Mon Sep 17 00:00:00 2001 From: Seppo Yli-Olli Date: Sun, 11 Dec 2022 16:23:52 +0200 Subject: [PATCH 3/6] Attempt at backporting regexp fixes See 782555a674de2ab6f1760f13c4a1b17400f64533 --- buildstream/element.py | 6 ++++-- buildstream/utils.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/buildstream/element.py b/buildstream/element.py index a83fe232a..2c38c8f8f 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -2311,7 +2311,9 @@ def __init_splits(self): bstdata = self.get_public_data('bst') splits = bstdata.get('split-rules') self.__splits = { - domain: re.compile('^(?:' + '|'.join([utils._glob2re(r) for r in rules]) + ')$') + domain: re.compile( + "^(?:" + "|".join([utils._glob2re(r) for r in rules]) + ")$", re.MULTILINE | re.DOTALL + ) for domain, rules in self.node_items(splits) } @@ -2386,7 +2388,7 @@ def __file_is_whitelisted(self, path): for index, exp in enumerate(whitelist) ] expression = ('^(?:' + '|'.join(whitelist_expressions) + ')$') - self.__whitelist_regex = re.compile(expression) + self.__whitelist_regex = re.compile(expression, re.MULTILINE | re.DOTALL) return self.__whitelist_regex.match(path) or self.__whitelist_regex.match(os.path.join(os.sep, path)) # __extract(): diff --git a/buildstream/utils.py b/buildstream/utils.py index b26d1f920..9f22e6301 100644 --- a/buildstream/utils.py +++ b/buildstream/utils.py @@ -200,7 +200,7 @@ def glob(paths, pattern): pattern = os.sep + pattern expression = _glob2re(pattern) - regexer = re.compile(expression) + regexer = re.compile(expression, re.MULTILINE | re.DOTALL) for filename in paths: filename_try = filename @@ -1138,7 +1138,7 @@ def resume_proc(): # def _glob2re(pat): i, n = 0, len(pat) - res = '(?ms)' + res = '' while i < n: c = pat[i] i = i + 1 From 1522e5343b26781e1a6bfb0e7b2b0a963cb4c0d9 Mon Sep 17 00:00:00 2001 From: Seppo Yli-Olli Date: Fri, 4 Nov 2022 00:04:52 +0200 Subject: [PATCH 4/6] Update pinned ruamel.yaml.clib to 0.2.7 The older version doesn't support Python 3.11 --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 9044ca1ba..39de8fb81 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -9,5 +9,5 @@ setuptools==44.1.1 ujson==5.4.0 ## The following requirements were added by pip freeze: MarkupSafe==2.1.1 -ruamel.yaml.clib==0.2.6 +ruamel.yaml.clib==0.2.7 six==1.16.0 From 3a5c24e9b8ae0126d1939d203deb689f504cd48b Mon Sep 17 00:00:00 2001 From: Seppo Yli-Olli Date: Sat, 12 Nov 2022 16:20:06 +0200 Subject: [PATCH 5/6] Bump tested grpcio to 1.51.1 This is first upstream release that advertises Python 3.11 support --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 39de8fb81..c9a8f0a1b 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,5 @@ click==8.1.3 -grpcio==1.48.0 +grpcio==1.51.1 Jinja2==3.1.2 pluginbase==1.0.1 protobuf==4.21.4 From 03acacf213f2b35ef5ff0fb4d23e3f38441ef7b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Thu, 15 Dec 2022 00:09:14 +0100 Subject: [PATCH 6/6] Update requirements --- requirements/cov-requirements.txt | 7 ++----- requirements/dev-requirements.txt | 28 ++++++++++++---------------- requirements/requirements.txt | 10 ++++------ 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/requirements/cov-requirements.txt b/requirements/cov-requirements.txt index c092d2344..20c72779c 100644 --- a/requirements/cov-requirements.txt +++ b/requirements/cov-requirements.txt @@ -3,9 +3,6 @@ pytest-cov==2.10.1 ## The following requirements were added by pip freeze: attrs==22.1.0 iniconfig==1.1.1 -packaging==21.3 +packaging==22.0 pluggy==1.0.0 -py==1.11.0 -pyparsing==3.0.9 -pytest==7.1.2 -tomli==2.0.1 +pytest==7.2.0 diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 84d78261c..ec58fa240 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,27 +1,23 @@ pep8==1.7.1 -pylint==2.14.5 -pytest==7.1.2 +pylint==2.15.8 +pytest==7.2.0 pytest-datafiles==2.0.1 -pytest-env==0.6.2 -pytest-xdist==2.5.0 +pytest-env==0.8.1 +pytest-xdist==3.1.0 pytest-timeout==2.1.0 -pyftpdlib==1.5.6 +pyftpdlib==1.5.7 ## The following requirements were added by pip freeze: -astroid==2.11.7 +astroid==2.12.13 attrs==22.1.0 -dill==0.3.5.1 +dill==0.3.6 execnet==1.9.0 iniconfig==1.1.1 -isort==5.10.1 -lazy-object-proxy==1.7.1 +isort==5.11.2 +lazy-object-proxy==1.8.0 mccabe==0.7.0 -packaging==21.3 -platformdirs==2.5.2 +packaging==22.0 +platformdirs==2.6.0 pluggy==1.0.0 py==1.11.0 -pyparsing==3.0.9 -pytest-forked==1.4.0 -tomli==2.0.1 -tomlkit==0.11.1 -typing-extensions==4.3.0 +tomlkit==0.11.6 wrapt==1.14.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index c9a8f0a1b..b0e7803c3 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,12 +2,10 @@ click==8.1.3 grpcio==1.51.1 Jinja2==3.1.2 pluginbase==1.0.1 -protobuf==4.21.4 -psutil==5.9.1 +protobuf==4.21.12 +psutil==5.9.4 ruamel.yaml==0.17.21 -setuptools==44.1.1 -ujson==5.4.0 +setuptools==62.6.0 +ujson==5.6.0 ## The following requirements were added by pip freeze: MarkupSafe==2.1.1 -ruamel.yaml.clib==0.2.7 -six==1.16.0