Add support for python 3.11

.github/common.env

@@ -1,5 +1,5 @@
 # Shared common variables
 
-CI_IMAGE_VERSION=master-533491591
-CI_TOXENV_MAIN=py36-nocover,py37-nocover,py38-nocover,py39-nocover,py310-nocover
+CI_IMAGE_VERSION=master-784208155
+CI_TOXENV_MAIN=py36-nocover,py37-nocover,py38-nocover,py39-nocover,py310-nocover,py311-nocover
 CI_TOXENV_ALL="${CI_TOXENV_MAIN}"

.github/compose/ci.docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.4'
 
 x-tests-template: &tests-template
-    image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:35-${CI_IMAGE_VERSION:-latest}
+    image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:36-${CI_IMAGE_VERSION:-latest}
     command: tox -vvvvv -- --color=yes --integration
     environment:
       TOXENV: ${CI_TOXENV_ALL}
@@ -22,14 +22,14 @@ x-tests-template: &tests-template
 
 services:
 
-  fedora-35:
-    <<: *tests-template
-    image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:35-${CI_IMAGE_VERSION:-latest}
-
   fedora-36:
     <<: *tests-template
     image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:36-${CI_IMAGE_VERSION:-latest}
 
+  fedora-37:
+    <<: *tests-template
+    image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:37-${CI_IMAGE_VERSION:-latest}
+
   debian-10:
     <<: *tests-template
     image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:10-${CI_IMAGE_VERSION:-latest}

.github/run-ci.sh

@@ -67,8 +67,8 @@ set -e
 if [ -z "${test_names}" ]; then
     runTest "lint"
     runTest "debian-10"
-    runTest "fedora-35"
     runTest "fedora-36"
+    runTest "fedora-37"
 else
     for test_name in "${test_names}"; do
 	runTest "${test_name}"

.github/workflows/ci.yml

@@ -26,8 +26,8 @@ jobs:
         # "../compose/ci.docker-compose.yml"
         test-name:
           - debian-10
-          - fedora-35
           - fedora-36
+          - fedora-37
           - lint
 
     steps:

buildstream/_exceptions.py

@@ -24,25 +24,10 @@
 # pylint: disable=global-statement
 
 # The last raised exception, this is used in test cases only
-_last_exception = None
 _last_task_error_domain = None
 _last_task_error_reason = None
 
 
-# get_last_exception()
-#
-# Fetches the last exception from the main process
-#
-# Used by regression tests
-#
-def get_last_exception():
-    global _last_exception
-
-    le = _last_exception
-    _last_exception = None
-    return le
-
-
 # get_last_task_error()
 #
 # Fetches the last exception from a task
@@ -102,7 +87,6 @@ class ErrorDomain(Enum):
 class BstError(Exception):
 
     def __init__(self, message, *, detail=None, domain=None, reason=None, temporary=False):
-        global _last_exception
 
         super().__init__(message)
 
@@ -126,9 +110,6 @@ def __init__(self, message, *, detail=None, domain=None, reason=None, temporary=
         self.domain = domain
         self.reason = reason
 
-        # Hold on to the last raised exception for testing purposes
-        _last_exception = self
-
 
 # PluginError
 #

buildstream/_frontend/app.py

@@ -22,9 +22,11 @@
 import resource
 import traceback
 import datetime
+from enum import Enum
 from textwrap import TextWrapper
 from contextlib import contextmanager
 
+import ujson
 import click
 from click import UsageError
 
@@ -35,7 +37,7 @@
 from .._context import Context
 from .._platform import Platform
 from .._project import Project
-from .._exceptions import BstError, StreamError, LoadError, LoadErrorReason, AppError
+from .._exceptions import BstError, StreamError, LoadError, LoadErrorReason, AppError, get_last_task_error
 from .._message import Message, MessageType, unconditional_messages
 from .._stream import Stream
 from .._versions import BST_FORMAT_VERSION
@@ -682,6 +684,20 @@ def _error_exit(self, error, prefix=None):
             detail = '\n' + indent + indent.join(error.detail.splitlines(True))
             click.echo("{}".format(detail), err=True)
 
+        # Record machine readable errors in a tempfile for the test harness to read back
+        if 'BST_TEST_ERROR_CODES' in os.environ:
+            task_error_domain, task_error_reason = get_last_task_error ()
+            error_codes = ujson.dumps ({
+                'main_error_domain': error.domain.value if error.domain else None,
+                'main_error_reason': error.reason.value if isinstance (error.reason, Enum) else error.reason,
+                'task_error_domain': task_error_domain.value if task_error_domain else None,
+                'task_error_reason': (
+                    task_error_reason.value if isinstance (task_error_reason, Enum) else task_error_reason
+                )
+            })
+            with open (os.environ['BST_TEST_ERROR_CODES'], "w", encoding="utf-8") as f:
+                f.write (error_codes)
+
         sys.exit(-1)
 
     #

buildstream/plugins/sources/pip.py

@@ -93,6 +93,7 @@
     'python3.8',
     'python3.9',
     'python3.10',
+    'python3.11',
 ]
 
 # List of allowed extensions taken from

requirements/requirements.txt

@@ -1,5 +1,5 @@
 click==8.1.3
-grpcio==1.48.0
+grpcio==1.51.1
 Jinja2==3.1.2
 pluginbase==1.0.1
 protobuf==4.21.4
@@ -9,5 +9,5 @@ setuptools==44.1.1
 ujson==5.4.0
 ## The following requirements were added by pip freeze:
 MarkupSafe==2.1.1
-ruamel.yaml.clib==0.2.6
+ruamel.yaml.clib==0.2.7
 six==1.16.0

setup.py

@@ -263,6 +263,8 @@ def get_cmdclass():
           'Programming Language :: Python :: 3.7',
           'Programming Language :: Python :: 3.8',
           'Programming Language :: Python :: 3.9',
+          'Programming Language :: Python :: 3.10',
+          'Programming Language :: Python :: 3.11',
           'Topic :: Software Development :: Build Tools'
       ],
       description='A framework for modelling build pipelines in YAML',

tests/artifactcache/expiry.py

@@ -289,6 +289,7 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
 # has 10K total disk space, and 6K of it is already in use (not
 # including any space used by the artifact cache).
 #
+@pytest.mark.xfail(reason="unittest.mock() not supported when running tests in subprocesses")
 @pytest.mark.parametrize("quota,err_domain,err_reason", [
     # Valid configurations
     ("1", 'success', None),

tests/format/assertion.py

@@ -32,4 +32,4 @@ def test_assertion_cli(cli, datafiles, target, opt_pony, opt_horsy, assertion):
 
     # Assert that the assertion text provided by the user
     # is found in the exception text
-    assert assertion in str(result.exception)
+    assert assertion in str(result.stderr)

tests/format/optionarch.py

@@ -12,6 +12,12 @@
 # Context manager to override the reported value of `os.uname()`
 @contextmanager
 def override_uname_arch(name):
+
+    #
+    # Disabling this test since we now run bst in a subprocess during tests.
+    #
+    pytest.xfail("Overriding os.uname() in bst subprocess is unsupported")
+
     orig_uname = os.uname
     orig_tuple = tuple(os.uname())
     override_result = (orig_tuple[0], orig_tuple[1],

tests/frontend/fetch.py

@@ -71,8 +71,7 @@ def test_fetch_consistency_bug(cli, tmpdir, datafiles):
     #    more gracefully as a BUG message.
     #
     result = cli.run(project=project, args=['fetch', 'bug.bst'])
-    assert result.exc is not None
-    assert str(result.exc) == "Something went terribly wrong"
+    assert "Something went terribly wrong" in result.stderr
 
 
 @pytest.mark.datafiles(DATA_DIR)

tests/frontend/show.py

@@ -269,6 +269,7 @@ def test_fetched_junction(cli, tmpdir, datafiles, element_name, workspaced):
 ###############################################################
 #                   Testing recursion depth                   #
 ###############################################################
+@pytest.mark.xfail(reason="recursion errors not currently detectable")
 @pytest.mark.parametrize("dependency_depth", [100, 500, 1200])
 def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth):
     project_name = "recursion-test"
@@ -314,8 +315,14 @@ def setup_test():
     if dependency_depth <= recursion_limit:
         result.assert_success()
     else:
-        #  Assert exception is thown and handled
-        assert not result.unhandled_exception
+        # XXX Assert exception is thown and handled
+        #
+        # We need to assert that the client has not thrown a stack trace for
+        # a recursion error, this should be done by creating a BstError instead
+        # of just handling it in app.py and doing sys.exit(), because we no longer
+        # have any way of detecting whether the client has thrown an exception
+        # otherwise
+        #
         assert result.exit_code == -1
 
     shutil.rmtree(project_path)

tests/integration/symlinks.py

@@ -70,5 +70,5 @@ def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, tmpdir, datafiles
     # point outside the sandbox which BuildStream needs to detect before it
     # tries to actually write there.
     result = cli.run(project=project, args=['checkout', element_name, checkout])
-    assert result.exit_code == -1
+    assert result.exit_code != 0
     assert "Destination path resolves to a path outside of the staging area" in result.stderr

tests/loader/junctions.py

@@ -135,21 +135,15 @@ def test_nested_conflict(cli, datafiles):
     copy_subprojects(project, datafiles, ['foo', 'bar'])
 
     result = cli.run(project=project, args=['build', 'target.bst'])
-    assert result.exit_code != 0
-    assert result.exception
-    assert isinstance(result.exception, LoadError)
-    assert result.exception.reason == LoadErrorReason.CONFLICTING_JUNCTION
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CONFLICTING_JUNCTION)
 
 
 @pytest.mark.datafiles(DATA_DIR)
 def test_invalid_missing(cli, datafiles):
     project = os.path.join(str(datafiles), 'invalid')
 
     result = cli.run(project=project, args=['build', 'missing.bst'])
-    assert result.exit_code != 0
-    assert result.exception
-    assert isinstance(result.exception, LoadError)
-    assert result.exception.reason == LoadErrorReason.MISSING_FILE
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -158,10 +152,7 @@ def test_invalid_with_deps(cli, datafiles):
     copy_subprojects(project, datafiles, ['base'])
 
     result = cli.run(project=project, args=['build', 'junction-with-deps.bst'])
-    assert result.exit_code != 0
-    assert result.exception
-    assert isinstance(result.exception, ElementError)
-    assert result.exception.reason == 'element-forbidden-depends'
+    result.assert_main_error(ErrorDomain.ELEMENT, 'element-forbidden-depends')
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -170,10 +161,7 @@ def test_invalid_junction_dep(cli, datafiles):
     copy_subprojects(project, datafiles, ['base'])
 
     result = cli.run(project=project, args=['build', 'junction-dep.bst'])
-    assert result.exit_code != 0
-    assert result.exception
-    assert isinstance(result.exception, LoadError)
-    assert result.exception.reason == LoadErrorReason.INVALID_DATA
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -248,10 +236,7 @@ def test_git_show(cli, tmpdir, datafiles):
 
     # Verify that bst show does not implicitly fetch subproject
     result = cli.run(project=project, args=['show', 'target.bst'])
-    assert result.exit_code != 0
-    assert result.exception
-    assert isinstance(result.exception, LoadError)
-    assert result.exception.reason == LoadErrorReason.SUBPROJECT_FETCH_NEEDED
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_FETCH_NEEDED)
 
     # Explicitly fetch subproject
     result = cli.run(project=project, args=['fetch', 'base.bst'])

tests/sources/tar.py

@@ -348,6 +348,7 @@ def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
 
 # Test that BuildStream doesnt crash if HOME is unset while
 # the netrc module is trying to find it's ~/.netrc file.
+@pytest.mark.xfail(reason="Cannot set environment variable to None when running tests in subprocesses")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
 def test_homeless_environment(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)