diff --git a/.pylintrc b/.pylintrc index 56aa38c1a..382c81b5c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -5,10 +5,8 @@ # run arbitrary code extension-pkg-whitelist= buildstream.node, - buildstream._loader._loader, buildstream._loader.loadelement, buildstream._loader.types, - buildstream._scheduler.jobs._job, buildstream._types, buildstream._utils, buildstream._variables, diff --git a/NEWS b/NEWS index b9dee6abf..989d587c8 100644 --- a/NEWS +++ b/NEWS @@ -43,6 +43,15 @@ CLI o BREAKING CHANGE: `bst shell --use-buildtree` is now a boolean flag. `--use-buildtree=ask` and `--use-buildtree=try` are no longer supported. + o BREAKING CHANGE: `--remote` options removed, replaced by `--artifact-remote` or `--source-remote` + + o BREAKING CHANGE: All old obsolete/deprecated commands removed, including: + - bst fetch (now bst source fetch) + - bst track (now bst source track) + - bst checkout (now bst artifact checkout) + - bst pull (now bst artifact pull) + - bst push (now bst artifact push) + ================== buildstream 1.93.5 diff --git a/doc/source/using_commands.rst b/doc/source/using_commands.rst index 1162e6c44..6f4b1ce96 100644 --- a/doc/source/using_commands.rst +++ b/doc/source/using_commands.rst @@ -9,6 +9,77 @@ invoked on the command line, where, in most cases, this will be from the project's main directory. +Commonly used parameters +------------------------ + + +.. _invoking_specify_remotes: + +Remotes +~~~~~~~ +Remote :ref:`cache servers ` can be specified on the +command line for commands which may result in communicating with such servers. + +Any command which has arguments to specify a ``REMOTE``, such as ``--artifact-remote`` +or ``--source-remote``, will override whatever was set in the user configuration, +and will have an accompanying switch which allows the command to decide whether +to ignore any remote :ref:`artifact ` or :ref:`source ` +caches suggested by project configuration. + +Remotes can be specified on the command line either as a simple URI, or as +a comma separated list of key value pairs. + +**Specifying a remote using a URI** + +.. code:: shell + + bst artifact push --remote https://artifacts.com/artifacts:8088 element.bst + +**Specifying a remote using key value pairs** + +.. code:: shell + + bst build --artifact-remote \ + url=https://artifacts.com/artifacts:8088,type=index,server-cert=~/artifacts.cert \ + element.bst + + +Attributes +'''''''''' +Here is the list attributes which can be spefied when providing a ``REMOTE`` on the command line: + +* ``url`` + + The URL of the remote, possibly including a port number. + +* ``instance-name`` + + The instance name of this remote, used for sharding by some implementations. + +* ``type`` + + Whether this remote is to be used for indexing, storage or both, as explained + in the corresponding :ref:`user configuration documentation ` + +* ``push`` + + Normally one need not specify this, as it is often inferred by the command + being used. In some cases, like :ref:`bst build `, it can + be useful to specify multiple remotes, and only allow pushing to some of + the remotes. + + If unspecified, this is assumed to be ``True`` and BuildStream will attempt + to push to the remote, but fallback to only pulling if insufficient credentials + were provided. + +* ``server-cert``, ``client-cert``, ``client-key``: + + These keys specify the attributes of the :ref:`authentication configuration `. + + When specifying these on the command line, they are interpreted as paths relative + to the current working directory. + + Top-level commands ------------------ diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst index 053d034d7..4f7b98e4f 100644 --- a/doc/source/using_config.rst +++ b/doc/source/using_config.rst @@ -225,8 +225,14 @@ Using artifact :ref:`cache servers ` is an essential means *build avoidance*, as it will allow you to avoid building an element which has already been built and uploaded to a common artifact server. -Artifact cache servers can be declared in three different ways, with differing -priorities. +Artifact cache servers can be declared in different ways, with differing priorities. + + +Command line +'''''''''''' +Various commands which involve connecting to artifact servers allow +:ref:`specifying remotes `, remotes specified +on the command line replace all user configuration. Global caches @@ -300,8 +306,14 @@ to be rebuilt because of changes in the dependency graph, as BuildStream will fi to download the source code from the cache server before attempting to obtain it from an external source, which may suffer higher latencies. -Source cache servers can be declared in three different ways, with differing -priorities. +Source cache servers can be declared in different ways, with differing priorities. + + +Command line +'''''''''''' +Various commands which involve connecting to source cache servers allow +:ref:`specifying remotes `, remotes specified +on the command line replace all user configuration. Global caches diff --git a/setup.py b/setup.py index 18b5e42be..022ecfb06 100755 --- a/setup.py +++ b/setup.py @@ -317,9 +317,7 @@ def files_from_module(modname): BUILD_EXTENSIONS = [] register_cython_module("buildstream.node") -register_cython_module("buildstream._loader._loader") register_cython_module("buildstream._loader.loadelement", dependencies=["buildstream.node"]) -register_cython_module("buildstream._scheduler.jobs._job") register_cython_module("buildstream._yaml", dependencies=["buildstream.node"]) register_cython_module("buildstream._types") register_cython_module("buildstream._utils") diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 9422a2837..24666504c 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -46,6 +46,26 @@ # pylint: enable=cyclic-import +# _CacheConfig +# +# A convenience object for parsing artifact/source cache configurations +# +class _CacheConfig: + def __init__(self, override_projects: bool, remote_specs: List[RemoteSpec]): + self.override_projects: bool = override_projects + self.remote_specs: List[RemoteSpec] = remote_specs + + @classmethod + def new_from_node(cls, node: MappingNode) -> "_CacheConfig": + node.validate_keys(["override-project-caches", "servers"]) + servers = node.get_sequence("servers", default=[], allowed_types=[MappingNode]) + + override_projects: bool = node.get_bool("push", default=False) + remote_specs: List[RemoteSpec] = [RemoteSpec.new_from_node(node) for node in servers] + + return cls(override_projects, remote_specs) + + # Context() # # The Context object holds all of the user preferences @@ -502,15 +522,19 @@ def get_toplevel_project(self) -> "Project": # Args: # connect_artifact_cache: Whether to try to contact remote artifact caches # connect_source_cache: Whether to try to contact remote source caches - # artifact_remote: An overriding artifact cache remote, or None - # source_remote: An overriding source cache remote, or None + # artifact_remotes: Artifact cache remotes specified on the commmand line + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects + # ignore_project_source_remotes: Whether to ignore artifact remotes specified by projects # def initialize_remotes( self, connect_artifact_cache: bool, connect_source_cache: bool, - artifact_remote: Optional[RemoteSpec], - source_remote: Optional[RemoteSpec], + artifact_remotes: Iterable[RemoteSpec] = (), + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ignore_project_source_remotes: bool = False, ) -> None: # Ensure all projects are fully loaded. @@ -528,34 +552,6 @@ def initialize_remotes( if remote_execution: self.pull_artifact_files, self.remote_execution_specs = self._load_remote_execution(remote_execution) - cli_artifact_remotes = [artifact_remote] if artifact_remote else [] - cli_source_remotes = [source_remote] if source_remote else [] - - # - # Helper function to resolve which remote specs apply for a given project - # - def resolve_specs_for_project( - project: "Project", global_config: _CacheConfig, override_key: str, project_attribute: str, - ) -> List[RemoteSpec]: - - # Obtain the overrides - override_node = self.get_overrides(project.name) - override_config_node = override_node.get_mapping(override_key, default={}) - override_config = _CacheConfig.new_from_node(override_config_node) - if override_config.override_projects: - return override_config.remote_specs - elif global_config.override_projects: - return global_config.remote_specs - - # If there were no explicit overrides, then take either the project specific - # config or fallback to the global config, and tack on the project recommended - # remotes at the end. - # - config_specs = override_config.remote_specs or global_config.remote_specs - project_specs = getattr(project, project_attribute) - all_specs = config_specs + project_specs - return list(utils._deduplicate(all_specs)) - # # Maintain our list of remote specs for artifact and source caches # @@ -564,13 +560,22 @@ def resolve_specs_for_project( source_specs: List[RemoteSpec] = [] if connect_artifact_cache: - artifact_specs = cli_artifact_remotes or resolve_specs_for_project( - project, self._global_artifact_cache_config, "artifacts", "artifact_cache_specs", + artifact_specs = self._resolve_specs_for_project( + project, + artifact_remotes, + ignore_project_artifact_remotes, + self._global_artifact_cache_config, + "artifacts", + "artifact_cache_specs", ) - if connect_source_cache: - source_specs = cli_source_remotes or resolve_specs_for_project( - project, self._global_source_cache_config, "source-caches", "source_cache_specs", + source_specs = self._resolve_specs_for_project( + project, + source_remotes, + ignore_project_source_remotes, + self._global_source_cache_config, + "source-caches", + "source_cache_specs", ) # Advertize the per project remote specs publicly for the frontend @@ -683,6 +688,64 @@ def get_cascache(self) -> CASCache: # Private methods # ###################################################### + # _resolve_specs_for_project() + # + # Helper function to resolve which remote specs apply for a given project + # + # Args: + # project: The project + # cli_remotes: The remotes specified in the CLI + # cli_override: Whether the CLI decided to override project suggestions + # global_config: The global user configuration for this remote type + # override_key: The key to lookup project overrides for this remote type + # project_attribute: The Project attribute for project suggestions + # + # Returns: + # The resolved remotes for this project. + # + def _resolve_specs_for_project( + self, + project: "Project", + cli_remotes: Iterable[RemoteSpec], + cli_override: bool, + global_config: _CacheConfig, + override_key: str, + project_attribute: str, + ) -> List[RemoteSpec]: + + # Early return if the CLI is taking full control + if cli_override and cli_remotes: + return list(cli_remotes) + + # Obtain the overrides + override_node = self.get_overrides(project.name) + override_config_node = override_node.get_mapping(override_key, default={}) + override_config = _CacheConfig.new_from_node(override_config_node) + + # + # Decide on what remotes to use from user config, if any + # + # Priority CLI -> Project overrides -> Global config + # + remotes: List[RemoteSpec] + if cli_remotes: + remotes = list(cli_remotes) + elif override_config.remote_specs: + remotes = override_config.remote_specs + else: + remotes = global_config.remote_specs + + # If any of the configs have disabled project remotes, return now + # + if cli_override or override_config.override_projects or global_config.override_projects: + return remotes + + # If there are any project recommendations, append them at the end + project_remotes = getattr(project, project_attribute) + remotes = list(utils._deduplicate(remotes + project_remotes)) + + return remotes + # Force the resolved XDG variables into the environment, # this is so that they can be used directly to specify # preferred locations of things from user configuration @@ -710,23 +773,3 @@ def _load_remote_execution(self, node: MappingNode) -> Tuple[bool, Optional[Remo remote_execution_specs = None return pull_artifact_files, remote_execution_specs - - -# _CacheConfig -# -# A convenience object for parsing artifact/source cache configurations -# -class _CacheConfig: - def __init__(self, override_projects: bool, remote_specs: List[RemoteSpec]): - self.override_projects: bool = override_projects - self.remote_specs: List[RemoteSpec] = remote_specs - - @classmethod - def new_from_node(cls, node: MappingNode) -> "_CacheConfig": - node.validate_keys(["override-project-caches", "servers"]) - servers = node.get_sequence("servers", default=[], allowed_types=[MappingNode]) - - override_projects: bool = node.get_bool("push", default=False) - remote_specs: List[RemoteSpec] = [RemoteSpec.new_from_node(node) for node in servers] - - return cls(override_projects, remote_specs) diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index ce1610fbc..c2afed938 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -711,7 +711,7 @@ def _handle_failure(self, element, task, failure): _Scope.BUILD, self.shell_prompt, isolate=True, - usebuildtree="always", + usebuildtree=True, unique_id=unique_id, ) except BstError as e: diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index ab06e8a8a..b70a13279 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -5,9 +5,10 @@ import shutil import click from .. import _yaml -from .._exceptions import BstError, LoadError, AppError +from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled -from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection +from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope +from .._remotespec import RemoteSpec, RemoteSpecPurpose from ..utils import UtilError @@ -36,6 +37,25 @@ def convert(self, value, param, ctx): return self._enum(super().convert(value, param, ctx)) +class RemoteSpecType(click.ParamType): + name = "remote" + + def __init__(self, purpose=RemoteSpecPurpose.ALL): + self.purpose = purpose + + def convert(self, value, param, ctx): + spec = None + try: + spec = RemoteSpec.new_from_string(value, self.purpose) + except RemoteError as e: + self.fail("Failed to interpret remote: {}".format(e)) + + return spec + + def __repr__(self): + return "REMOTE" + + ################################################################## # Override of click's main entry point # ################################################################## @@ -412,11 +432,38 @@ def init(app, project_name, min_version, element_path, force, target_directory): help="The dependencies to build", ) @click.option( - "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)" + "--artifact-remote", + "artifact_remotes", + type=RemoteSpecType(), + multiple=True, + help="A remote for uploading and downloading artifacts", +) +@click.option( + "--source-remote", + "source_remotes", + type=RemoteSpecType(), + multiple=True, + help="A remote for uploading and downloading cached sources", +) +@click.option( + "--ignore-project-artifact-remotes", + is_flag=True, + help="Ignore remote artifact cache servers recommended by projects", +) +@click.option( + "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def build(app, elements, deps, remote): +def build( + app, + elements, + deps, + artifact_remotes, + source_remotes, + ignore_project_artifact_remotes, + ignore_project_source_remotes, +): """Build elements in a pipeline Specifying no elements will result in building the default targets @@ -444,7 +491,15 @@ def build(app, elements, deps, remote): # Junction elements cannot be built, exclude them from default targets ignore_junction_targets = True - app.stream.build(elements, selection=deps, ignore_junction_targets=ignore_junction_targets, remote=remote) + app.stream.build( + elements, + selection=deps, + ignore_junction_targets=ignore_junction_targets, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) ################################################################## @@ -586,10 +641,45 @@ def show(app, elements, deps, except_, order, format_): ), ) @click.option("--pull", "pull_", is_flag=True, help="Attempt to pull missing or incomplete artifacts") +@click.option( + "--artifact-remote", + "artifact_remotes", + type=RemoteSpecType(), + multiple=True, + help="A remote for uploading and downloading artifacts", +) +@click.option( + "--source-remote", + "source_remotes", + type=RemoteSpecType(), + multiple=True, + help="A remote for uploading and downloading cached sources", +) +@click.option( + "--ignore-project-artifact-remotes", + is_flag=True, + help="Ignore remote artifact cache servers recommended by projects", +) +@click.option( + "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" +) @click.argument("element", required=False, type=click.Path(readable=False)) @click.argument("command", type=click.STRING, nargs=-1) @click.pass_obj -def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command): +def shell( + app, + element, + command, + mount, + isolate, + build_, + cli_buildtree, + pull_, + artifact_remotes, + source_remotes, + ignore_project_artifact_remotes, + ignore_project_source_remotes, +): """Run a command in the target element's sandbox environment When this command is executed from a workspace directory, the default @@ -611,8 +701,6 @@ def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command): If no COMMAND is specified, the default is to attempt to run an interactive shell. """ - from ..element import _Scope - from .._project import HostMount # Buildtree can only be used with build shells if cli_buildtree: @@ -626,7 +714,7 @@ def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command): if not element: raise AppError('Missing argument "ELEMENT".') - mounts = [HostMount(path, host_path) for host_path, path in mount] + mounts = [_HostMount(path, host_path) for host_path, path in mount] try: exitcode = app.stream.shell( @@ -638,6 +726,10 @@ def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command): command=command, usebuildtree=cli_buildtree, pull_=pull_, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, ) except BstError as e: raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e @@ -683,11 +775,18 @@ def source(): help="The dependencies to fetch", ) @click.option( - "--remote", "-r", default=None, help="The URL of the remote source cache (defaults to the first configured cache)" + "--source-remote", + "source_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PULL), + multiple=True, + help="A remote for downloading sources", +) +@click.option( + "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def source_fetch(app, elements, deps, except_, remote): +def source_fetch(app, elements, deps, except_, source_remotes, ignore_project_source_remotes): """Fetch sources required to build the pipeline Specifying no elements will result in fetching the default targets @@ -715,7 +814,13 @@ def source_fetch(app, elements, deps, except_, remote): if not elements: elements = app.project.get_default_targets() - app.stream.fetch(elements, selection=deps, except_targets=except_, remote=remote) + app.stream.fetch( + elements, + selection=deps, + except_targets=except_, + source_remotes=source_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) ################################################################## @@ -740,11 +845,18 @@ def source_fetch(app, elements, deps, except_, remote): help="The dependencies to push", ) @click.option( - "--remote", "-r", default=None, help="The URL of the remote source cache (defaults to the first configured cache)" + "--source-remote", + "source_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PUSH), + multiple=True, + help="A remote for uploading sources", +) +@click.option( + "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def source_push(app, elements, deps, remote): +def source_push(app, elements, deps, source_remotes, ignore_project_source_remotes): """Push sources required to build the pipeline Specifying no elements will result in pushing the sources of the default @@ -767,7 +879,12 @@ def source_push(app, elements, deps, remote): if not elements: elements = app.project.get_default_targets() - app.stream.source_push(elements, selection=deps, remote=remote) + app.stream.source_push( + elements, + selection=deps, + source_remotes=source_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) ################################################################## @@ -867,9 +984,31 @@ def source_track(app, elements, deps, except_, cross_junctions): type=click.Path(file_okay=False), help="The directory to checkout the sources to", ) +@click.option( + "--source-remote", + "source_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PULL), + multiple=True, + help="A remote for downloading cached sources", +) +@click.option( + "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" +) @click.argument("element", required=False, type=click.Path(readable=False)) @click.pass_obj -def source_checkout(app, element, directory, force, deps, except_, tar, compression, build_scripts): +def source_checkout( + app, + element, + directory, + force, + deps, + except_, + tar, + compression, + build_scripts, + source_remotes, + ignore_project_source_remotes, +): """Checkout sources of an element to the specified location When this command is executed from a workspace directory, the default @@ -903,6 +1042,8 @@ def source_checkout(app, element, directory, force, deps, except_, tar, compress tar=bool(tar), compression=compression, include_build_scripts=build_scripts, + source_remotes=source_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, ) @@ -932,13 +1073,30 @@ def workspace(): default=None, help="Only for use when a single Element is given: Set the directory to use to create the workspace", ) +@click.option( + "--source-remote", + "source_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PULL), + multiple=True, + help="A remote for downloading cached sources", +) +@click.option( + "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" +) @click.argument("elements", nargs=-1, type=click.Path(readable=False), required=True) @click.pass_obj -def workspace_open(app, no_checkout, force, directory, elements): +def workspace_open(app, no_checkout, force, directory, source_remotes, ignore_project_source_remotes, elements): """Open a workspace for manual source modification""" with app.initialized(): - app.stream.workspace_open(elements, no_checkout=no_checkout, force=force, custom_dir=directory) + app.stream.workspace_open( + elements, + no_checkout=no_checkout, + force=force, + custom_dir=directory, + source_remotes=source_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) ################################################################## @@ -1126,9 +1284,34 @@ def artifact_show(app, deps, artifacts): @click.option( "--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to" ) +@click.option( + "--artifact-remote", + "artifact_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PULL), + multiple=True, + help="A remote for downloading artifacts", +) +@click.option( + "--ignore-project-artifact-remotes", + is_flag=True, + help="Ignore remote artifact cache servers recommended by projects", +) @click.argument("target", required=False, type=click.Path(readable=False)) @click.pass_obj -def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, pull_, directory, target): +def artifact_checkout( + app, + force, + deps, + integrate, + hardlinks, + tar, + compression, + pull_, + directory, + artifact_remotes, + ignore_project_artifact_remotes, + target, +): """Checkout contents of an artifact When this command is executed from a workspace directory, the default @@ -1188,6 +1371,8 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, pull=pull_, compression=compression, tar=bool(tar), + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) @@ -1207,11 +1392,20 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, help="The dependency artifacts to pull", ) @click.option( - "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)" + "--artifact-remote", + "artifact_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PULL), + multiple=True, + help="A remote for downloading artifacts", +) +@click.option( + "--ignore-project-artifact-remotes", + is_flag=True, + help="Ignore remote artifact cache servers recommended by projects", ) @click.argument("artifacts", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def artifact_pull(app, artifacts, deps, remote): +def artifact_pull(app, deps, artifact_remotes, ignore_project_artifact_remotes, artifacts): """Pull a built artifact from the configured remote artifact cache. Specifying no elements will result in pulling the default targets @@ -1222,8 +1416,8 @@ def artifact_pull(app, artifacts, deps, remote): is to pull the workspace element. By default the artifact will be pulled one of the configured caches - if possible, following the usual priority order. If the `--remote` flag - is given, only the specified cache will be queried. + if possible, following the usual priority order. If the `--artifact-remote` + flag is given, only the specified cache will be queried. Specify `--deps` to control which artifacts to pull: @@ -1242,7 +1436,13 @@ def artifact_pull(app, artifacts, deps, remote): # Junction elements cannot be pulled, exclude them from default targets ignore_junction_targets = True - app.stream.pull(artifacts, selection=deps, remote=remote, ignore_junction_targets=ignore_junction_targets) + app.stream.pull( + artifacts, + selection=deps, + ignore_junction_targets=ignore_junction_targets, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ) ################################################################## @@ -1261,12 +1461,21 @@ def artifact_pull(app, artifacts, deps, remote): help="The dependencies to push", ) @click.option( - "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)" + "--artifact-remote", + "artifact_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PUSH), + multiple=True, + help="A remote for uploading artifacts", +) +@click.option( + "--ignore-project-artifact-remotes", + is_flag=True, + help="Ignore remote artifact cache servers recommended by projects", ) @click.argument("artifacts", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def artifact_push(app, artifacts, deps, remote): - """Push a built artifact to a remote artifact cache. +def artifact_push(app, deps, artifact_remotes, ignore_project_artifact_remotes, artifacts): + """Push built artifacts to a remote artifact cache, possibly pulling them first. Specifying no elements will result in pushing the default targets of the project. If no default targets are configured, all project @@ -1275,9 +1484,6 @@ def artifact_push(app, artifacts, deps, remote): When this command is executed from a workspace directory, the default is to push the workspace element. - The default destination is the highest priority configured cache. You can - override this by passing a different cache URL with the `--remote` flag. - If bst has been configured to include build trees on artifact pulls, an attempt will be made to pull any required build trees to avoid the skipping of partial artifacts being pushed. @@ -1298,7 +1504,13 @@ def artifact_push(app, artifacts, deps, remote): # Junction elements cannot be pushed, exclude them from default targets ignore_junction_targets = True - app.stream.push(artifacts, selection=deps, remote=remote, ignore_junction_targets=ignore_junction_targets) + app.stream.push( + artifacts, + selection=deps, + ignore_junction_targets=ignore_junction_targets, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ) ################################################################ @@ -1395,141 +1607,3 @@ def artifact_delete(app, artifacts, deps): """Remove artifacts from the local cache""" with app.initialized(): app.stream.artifact_delete(artifacts, selection=deps) - - -################################################################## -# DEPRECATED Commands # -################################################################## - -# XXX: The following commands are now obsolete, but they are kept -# here along with all the options so that we can provide nice error -# messages when they are called. -# Also, note that these commands are hidden from the top-level help. - -################################################################## -# Fetch Command # -################################################################## -@cli.command(short_help="COMMAND OBSOLETE - Fetch sources in a pipeline", hidden=True) -@click.option( - "--except", - "except_", - multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies from fetching", -) -@click.option( - "--deps", - "-d", - default=_PipelineSelection.PLAN, - show_default=True, - type=FastEnumType(_PipelineSelection, [_PipelineSelection.NONE, _PipelineSelection.PLAN, _PipelineSelection.ALL]), - help="The dependencies to fetch", -) -@click.argument("elements", nargs=-1, type=click.Path(readable=False)) -@click.pass_obj -def fetch(app, elements, deps, except_): - click.echo("This command is now obsolete. Use `bst source fetch` instead.", err=True) - sys.exit(1) - - -################################################################## -# Track Command # -################################################################## -@cli.command(short_help="COMMAND OBSOLETE - Track new source references", hidden=True) -@click.option( - "--except", - "except_", - multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies from tracking", -) -@click.option( - "--deps", - "-d", - default=_PipelineSelection.NONE, - show_default=True, - type=FastEnumType(_PipelineSelection, [_PipelineSelection.NONE, _PipelineSelection.ALL]), - help="The dependencies to track", -) -@click.option("--cross-junctions", "-J", is_flag=True, help="Allow crossing junction boundaries") -@click.argument("elements", nargs=-1, type=click.Path(readable=False)) -@click.pass_obj -def track(app, elements, deps, except_, cross_junctions): - click.echo("This command is now obsolete. Use `bst source track` instead.", err=True) - sys.exit(1) - - -################################################################## -# Checkout Command # -################################################################## -@cli.command(short_help="COMMAND OBSOLETE - Checkout a built artifact", hidden=True) -@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten") -@click.option( - "--deps", - "-d", - default=_PipelineSelection.RUN, - show_default=True, - type=FastEnumType(_PipelineSelection, [_PipelineSelection.RUN, _PipelineSelection.BUILD, _PipelineSelection.NONE]), - help="The dependencies to checkout", -) -@click.option("--integrate/--no-integrate", default=True, help="Run integration commands (default is to run commands)") -@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copies (handle with care)") -@click.option( - "--tar", - is_flag=True, - help="Create a tarball from the artifact contents instead " - "of a file tree. If LOCATION is '-', the tarball " - "will be dumped to the standard output.", -) -@click.argument("element", required=False, type=click.Path(readable=False)) -@click.argument("location", type=click.Path(), required=False) -@click.pass_obj -def checkout(app, element, location, force, deps, integrate, hardlinks, tar): - click.echo( - "This command is now obsolete. Use `bst artifact checkout` instead " - + "and use the --directory option to specify LOCATION", - err=True, - ) - sys.exit(1) - - -################################################################ -# Pull Command # -################################################################ -@cli.command(short_help="COMMAND OBSOLETE - Pull a built artifact", hidden=True) -@click.option( - "--deps", - "-d", - default=_PipelineSelection.NONE, - show_default=True, - type=FastEnumType(_PipelineSelection, [_PipelineSelection.NONE, _PipelineSelection.ALL]), - help="The dependency artifacts to pull", -) -@click.option("--remote", "-r", help="The URL of the remote cache (defaults to the first configured cache)") -@click.argument("elements", nargs=-1, type=click.Path(readable=False)) -@click.pass_obj -def pull(app, elements, deps, remote): - click.echo("This command is now obsolete. Use `bst artifact pull` instead.", err=True) - sys.exit(1) - - -################################################################## -# Push Command # -################################################################## -@cli.command(short_help="COMMAND OBSOLETE - Push a built artifact", hidden=True) -@click.option( - "--deps", - "-d", - default=_PipelineSelection.NONE, - show_default=True, - type=FastEnumType(_PipelineSelection, [_PipelineSelection.NONE, _PipelineSelection.ALL]), - help="The dependencies to push", -) -@click.option( - "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)" -) -@click.argument("elements", nargs=-1, type=click.Path(readable=False)) -@click.pass_obj -def push(app, elements, deps, remote): - click.echo("This command is now obsolete. Use `bst artifact push` instead.", err=True) - sys.exit(1) diff --git a/src/buildstream/_loader/_loader.pyi b/src/buildstream/_loader/_loader.pyi deleted file mode 100644 index c4281b4b9..000000000 --- a/src/buildstream/_loader/_loader.pyi +++ /dev/null @@ -1 +0,0 @@ -def valid_chars_name(name: str) -> bool: ... diff --git a/src/buildstream/_loader/_loader.pyx b/src/buildstream/_loader/_loader.pyx deleted file mode 100644 index 258a34ab7..000000000 --- a/src/buildstream/_loader/_loader.pyx +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (C) 2019 Bloomberg L.P. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . -# -# Authors: -# Benjamin Schubert -# - - -# Check if given filename containers valid characters. -# -# Args: -# name (str): Name of the file -# -# Returns: -# (bool): True if all characters are valid, False otherwise. -# -def valid_chars_name(str name): - cdef int char_value - cdef int forbidden_char - - for char_value in name: - # 0-31 are control chars, 127 is DEL, and >127 means non-ASCII - if char_value <= 31 or char_value >= 127: - return False - - # Disallow characters that are invalid on Windows. The list can be - # found at https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file - # - # Note that although : (colon) is not allowed, we do not raise - # warnings because of that, since we use it as a separator for - # junctioned elements. - # - # We also do not raise warnings on slashes since they are used as - # path separators. - for forbidden_char in '<>"|?*': - if char_value == forbidden_char: - return False - - return True diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 6ace3624b..48fb1c4e6 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -27,12 +27,12 @@ from ..node import Node from .._profile import Topics, PROFILER from .._includes import Includes +from .._utils import valid_chars_name +from ..types import CoreWarnings, _KeyStrength -from ._loader import valid_chars_name from .types import Symbol from . import loadelement from .loadelement import LoadElement, Dependency, DependencyType, extract_depends_from_node -from ..types import CoreWarnings, _KeyStrength # Loader(): diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index c8d8e5cfb..b54139ca3 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -35,7 +35,7 @@ from ._options import OptionPool from .node import ScalarNode, SequenceNode, MappingNode, ProvenanceInformation, _assert_symbol_name from ._pluginfactory import ElementFactory, SourceFactory, load_plugin_origin -from .types import CoreWarnings +from .types import CoreWarnings, _HostMount from ._projectrefs import ProjectRefs, ProjectRefStorage from ._loader import Loader, LoadContext from .element import Element @@ -52,27 +52,6 @@ _PROJECT_CONF_FILE = "project.conf" -# HostMount() -# -# A simple object describing the behavior of -# a host mount. -# -class HostMount: - def __init__(self, path, host_path=None, optional=False): - - # Support environment variable expansion in host mounts - path = os.path.expandvars(path) - if host_path is not None: - host_path = os.path.expandvars(host_path) - - self.path = path # Path inside the sandbox - self.host_path = host_path # Path on the host - self.optional = optional # Optional mounts do not incur warnings or errors - - if self.host_path is None: - self.host_path = self.path - - # Represents project configuration that can have different values for junctions. class ProjectConfig: def __init__(self): @@ -159,7 +138,7 @@ def __init__( self._fatal_warnings: List[str] = [] # A list of warnings which should trigger an error self._shell_command: List[str] = [] # The default interactive shell command self._shell_environment: Dict[str, str] = {} # Statically set environment vars - self._shell_host_files: List[str] = [] # A list of HostMount objects + self._shell_host_files: List[_HostMount] = [] # A list of HostMount objects # This is a lookup table of lists indexed by project, # the child dictionaries are lists of ScalarNodes indicating @@ -256,7 +235,7 @@ def translate_url(self, url, *, first_pass=False): # Returns: # (list): The shell command # (dict): The shell environment - # (list): The list of HostMount objects + # (list): The list of _HostMount objects # def get_shell_config(self): return (self._shell_command, self._shell_environment, self._shell_host_files) @@ -915,7 +894,7 @@ def _load_second_pass(self): host_files = shell_options.get_sequence("host-files", default=[]) for host_file in host_files: if isinstance(host_file, ScalarNode): - mount = HostMount(host_file.as_str()) + mount = _HostMount(host_file.as_str()) else: # Some validation host_file.validate_keys(["path", "host_path", "optional"]) @@ -924,7 +903,7 @@ def _load_second_pass(self): path = host_file.get_str("path") host_path = host_file.get_str("host_path", default=None) optional = host_file.get_bool("optional", default=False) - mount = HostMount(path, host_path, optional) + mount = _HostMount(path, host_path, optional) self._shell_host_files.append(mount) diff --git a/src/buildstream/_remotespec.py b/src/buildstream/_remotespec.py index e91fb5582..a37698e27 100644 --- a/src/buildstream/_remotespec.py +++ b/src/buildstream/_remotespec.py @@ -44,6 +44,20 @@ def __str__(self) -> str: return "" +# RemoteSpecPurpose(): +# +# What a RemoteSpec is going to be used for. +# +# This is currently only used to control the behavior +# of RemoteSpec.new_from_string(), after that, a RemoteSpec +# has a `push` attribute which is either True or False. +# +class RemoteSpecPurpose(FastEnum): + ALL = 0 # Pushing and pulling + PUSH = 1 # Only pushing + PULL = 2 # Only pulling + + # RemoteSpec(): # # This data structure holds all of the details required to @@ -252,6 +266,116 @@ def new_from_node( spec_node=spec_node, ) + # new_from_string(): + # + # Creates a RemoteSpec() from a string, used to parse CLI parameters + # + # If certificates are passed, they are interpreted as relative to the + # current working directory. + # + # Args: + # string: The user provided string + # purpose: The purpose this RemoteSpec is intended for (RemoteSpecPurpose) + # + # Returns: + # The described RemoteSpec instance. + # + # Raises: + # RemoteError: In case parsing the string fails + # + @classmethod + def new_from_string(cls, string: str, purpose: int = RemoteSpecPurpose.ALL) -> "RemoteSpec": + url: Optional[str] = None + instance_name: Optional[str] = None + remote_type: str = RemoteType.ALL + push: bool = True + server_cert: Optional[str] = None + client_key: Optional[str] = None + client_cert: Optional[str] = None + + if purpose == RemoteSpecPurpose.PULL: + push = False + + split = string.split(",") + if len(split) > 1: + for split_string in split: + subsplit = split_string.split("=") + + if len(subsplit) != 2: + raise RemoteError( + "Invalid format '{}' found in remote specification: {}".format(split_string, string) + ) + + key: str = subsplit[0] + val: str = subsplit[1] + + if key == "url": + url = val + elif key == "instance-name": + instance_name = val + elif key == "type": + remote_type = val + if remote_type not in [RemoteType.INDEX, RemoteType.STORAGE, RemoteType.ALL]: + raise RemoteError( + "Value for remote 'type' must be one of: {}".format( + ", ".join([RemoteType.INDEX, RemoteType.STORAGE, RemoteType.ALL]) + ) + ) + elif key == "push": + + # Provide a sensible error for `bst artifact push --remote url=http://pony.com,push=False ...` + if purpose != RemoteSpecPurpose.ALL: + raise RemoteError("The 'push' key is invalid and assumed to be {}".format(push)) + + if val in ("True", "true"): + push = True + elif val in ("False", "false"): + push = False + else: + raise RemoteError("Value for 'push' must be 'True' or 'False'") + elif key == "server-cert": + server_cert = cls._resolve_path(val, os.getcwd()) + elif key == "client-key": + client_key = cls._resolve_path(val, os.getcwd()) + elif key == "client-cert": + client_cert = cls._resolve_path(val, os.getcwd()) + else: + raise RemoteError("Unexpected key '{}' encountered".format(key)) + else: + # No commas, only the URL was specified + url = string + + if not url: + raise RemoteError("No URL specified in remote") + + return cls( + remote_type, + url, + push=push, + server_cert=server_cert, + client_key=client_key, + client_cert=client_cert, + instance_name=instance_name, + ) + + # _resolve_path() + # + # Resolve a path relative to the base directory + # + # Args: + # path: The path + # basedir: The base directory + # + # Returns: + # The resolved path + # + @classmethod + def _resolve_path(cls, path: str, basedir: Optional[str]) -> str: + path = os.path.expanduser(path) + if basedir: + path = os.path.join(basedir, path) + return path + # _parse_auth(): # # Parse the "auth" data @@ -276,9 +400,7 @@ def _parse_auth( for key in auth_keys: value = auth_node.get_str(key, None) if value: - value = os.path.expanduser(value) - if basedir: - value = os.path.join(basedir, value) + value = cls._resolve_path(value, basedir) auth_values[key] = value server_cert = auth_values["server-cert"] diff --git a/src/buildstream/_scheduler/jobs/_job.pyi b/src/buildstream/_scheduler/jobs/_job.pyi deleted file mode 100644 index fbf3e64de..000000000 --- a/src/buildstream/_scheduler/jobs/_job.pyi +++ /dev/null @@ -1 +0,0 @@ -def terminate_thread(thread_id: int): ... diff --git a/src/buildstream/_scheduler/jobs/_job.pyx b/src/buildstream/_scheduler/jobs/_job.pyx deleted file mode 100644 index 82f6ab044..000000000 --- a/src/buildstream/_scheduler/jobs/_job.pyx +++ /dev/null @@ -1,15 +0,0 @@ -from cpython.pystate cimport PyThreadState_SetAsyncExc -from cpython.ref cimport PyObject -from ..._signals import TerminateException - - -# terminate_thread() -# -# Ask a given a given thread to terminate by raising an exception in it. -# -# Args: -# thread_id (int): the thread id in which to throw the exception -# -def terminate_thread(long thread_id): - res = PyThreadState_SetAsyncExc(thread_id, TerminateException) - assert res == 1 diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py index aa71b6e18..c875afe2e 100644 --- a/src/buildstream/_scheduler/jobs/job.py +++ b/src/buildstream/_scheduler/jobs/job.py @@ -30,10 +30,10 @@ # BuildStream toplevel imports from ... import utils +from ..._utils import terminate_thread from ..._exceptions import ImplError, BstError, set_last_task_error, SkipJob from ..._message import Message, MessageType, unconditional_messages from ...types import FastEnum -from ._job import terminate_thread from ..._signals import TerminateException diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 1728bd75f..17011de53 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -29,7 +29,7 @@ import tempfile from contextlib import contextmanager, suppress from collections import deque -from typing import List, Tuple +from typing import List, Tuple, Optional, Iterable, Callable from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject @@ -47,9 +47,9 @@ from .element import Element from ._profile import Topics, PROFILER from ._project import ProjectRefStorage -from ._remotespec import RemoteType, RemoteSpec +from ._remotespec import RemoteSpec from ._state import State -from .types import _KeyStrength, _PipelineSelection, _Scope +from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount from .plugin import Plugin from . import utils, _yaml, _site, _pipeline @@ -135,32 +135,46 @@ def set_project(self, project): # and `bst shell`. # # Args: - # targets (list of str): Targets to pull - # selection (_PipelineSelection): The selection mode for the specified targets - # except_targets (list of str): Specified targets to except from fetching - # connect_artifact_cache (bool): Whether to try to contact remote artifact caches + # targets: Targets to pull + # selection: The selection mode for the specified targets (_PipelineSelection) + # except_targets: Specified targets to except from fetching # load_artifacts (bool): Whether to load artifacts with artifact names + # connect_artifact_cache: Whether to try to contact remote artifact caches + # connect_source_cache: Whether to try to contact remote source caches + # artifact_remotes: Artifact cache remotes specified on the commmand line + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects + # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # # Returns: # (list of Element): The selected elements def load_selection( self, - targets, + targets: Iterable[str], *, - selection=_PipelineSelection.NONE, - except_targets=(), - connect_artifact_cache=False, - load_artifacts=False, + selection: str = _PipelineSelection.NONE, + except_targets: Iterable[str] = (), + load_artifacts: bool = False, + connect_artifact_cache: bool = False, + connect_source_cache: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ignore_project_source_remotes: bool = False, ): with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)): target_objects = self._load( targets, selection=selection, except_targets=except_targets, - connect_artifact_cache=connect_artifact_cache, load_artifacts=load_artifacts, + connect_artifact_cache=connect_artifact_cache, + connect_source_cache=connect_source_cache, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, ) - return target_objects # shell() @@ -168,40 +182,58 @@ def load_selection( # Run a shell # # Args: - # element (str): The name of the element to run the shell for - # scope (_Scope): The scope for the shell (_Scope.BUILD or _Scope.RUN) - # prompt (function): A function to return the prompt to display in the shell - # mounts (list of HostMount): Additional directories to mount into the sandbox + # target: The name of the element to run the shell for + # scope: The scope for the shell, only BUILD or RUN are valid (_Scope) + # prompt: A function to return the prompt to display in the shell + # unique_id: (str): A unique_id to use to lookup an Element instance + # mounts: Additional directories to mount into the sandbox # isolate (bool): Whether to isolate the environment like we do in builds # command (list): An argv to launch in the sandbox, or None # usebuildtree (bool): Whether to use a buildtree as the source, given cli option # pull_ (bool): Whether to attempt to pull missing or incomplete artifacts - # unique_id: (str): Whether to use a unique_id to load an Element instance + # artifact_remotes: Artifact cache remotes specified on the commmand line + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects + # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # # Returns: # (int): The exit code of the launched shell # def shell( self, - element, - scope, - prompt, + target: str, + scope: int, + prompt: Callable[[Element], str], *, - mounts=None, - isolate=False, - command=None, - usebuildtree=False, - pull_=False, - unique_id=None, + unique_id: Optional[str] = None, + mounts: Optional[List[_HostMount]] = None, + isolate: bool = False, + command: Optional[List[str]] = None, + usebuildtree: bool = False, + pull_: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ignore_project_source_remotes: bool = False, ): + element: Element # Load the Element via the unique_id if given - if unique_id and element is None: + if unique_id and target is None: element = Plugin._lookup(unique_id) else: selection = _PipelineSelection.BUILD if scope == _Scope.BUILD else _PipelineSelection.RUN - elements = self.load_selection((element,), selection=selection, connect_artifact_cache=True) + elements = self.load_selection( + (target,), + selection=selection, + connect_artifact_cache=True, + connect_source_cache=True, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) # Get element to stage from `targets` list. # If scope is BUILD, it will not be in the `elements` list. @@ -260,24 +292,40 @@ def shell( # Builds (assembles) elements in the pipeline. # # Args: - # targets (list of str): Targets to build - # selection (_PipelineSelection): The selection mode for the specified targets - # ignore_junction_targets (bool): Whether junction targets should be filtered out - # remote (str): The URL of a specific remote server to push to, or None + # targets: Targets to build + # selection: The selection mode for the specified targets (_PipelineSelection) + # ignore_junction_targets: Whether junction targets should be filtered out + # artifact_remotes: Artifact cache remotes specified on the commmand line + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects + # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # # If `remote` specified as None, then regular configuration will be used # to determine where to push artifacts to. # - def build(self, targets, *, selection=_PipelineSelection.PLAN, ignore_junction_targets=False, remote=None): + def build( + self, + targets: Iterable[str], + *, + selection: str = _PipelineSelection.PLAN, + ignore_junction_targets: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ignore_project_source_remotes: bool = False, + ): elements = self._load( targets, selection=selection, ignore_junction_targets=ignore_junction_targets, + dynamic_plan=True, connect_artifact_cache=True, - artifact_remote_url=remote, connect_source_cache=True, - dynamic_plan=True, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, ) # Assert that the elements are consistent @@ -320,19 +368,29 @@ def build(self, targets, *, selection=_PipelineSelection.PLAN, ignore_junction_t # Fetches sources on the pipeline. # # Args: - # targets (list of str): Targets to fetch - # selection (_PipelineSelection): The selection mode for the specified targets - # except_targets (list of str): Specified targets to except from fetching - # remote (str|None): The URL of a specific remote server to pull from. + # targets: Targets to fetch + # selection: The selection mode for the specified targets (_PipelineSelection) + # except_targets: Specified targets to except from fetching + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # - def fetch(self, targets, *, selection=_PipelineSelection.PLAN, except_targets=None, remote=None): + def fetch( + self, + targets: Iterable[str], + *, + selection: str = _PipelineSelection.PLAN, + except_targets: Iterable[str] = (), + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_source_remotes: bool = False, + ): elements = self._load( targets, selection=selection, except_targets=except_targets, connect_source_cache=True, - source_remote_url=remote, + source_remotes=source_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, ) # Delegated to a shared fetch method @@ -376,7 +434,8 @@ def track(self, targets, *, selection=_PipelineSelection.REDIRECT, except_target # Args: # targets (list of str): Targets to push # selection (_PipelineSelection): The selection mode for the specified targets - # remote (str): The URL of a specific remote server to push to, or None + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # # If `remote` specified as None, then regular configuration will be used # to determine where to push sources to. @@ -385,10 +444,22 @@ def track(self, targets, *, selection=_PipelineSelection.REDIRECT, except_target # a fetch queue will be created if user context and available remotes allow for # attempting to fetch them. # - def source_push(self, targets, *, selection=_PipelineSelection.NONE, remote=None): + def source_push( + self, + targets, + *, + selection=_PipelineSelection.NONE, + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_source_remotes: bool = False, + ): elements = self._load( - targets, selection=selection, connect_source_cache=True, source_remote_url=remote, load_artifacts=True, + targets, + selection=selection, + load_artifacts=True, + connect_source_cache=True, + source_remotes=source_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, ) if not self._sourcecache.has_push_remotes(): @@ -408,24 +479,31 @@ def source_push(self, targets, *, selection=_PipelineSelection.NONE, remote=None # Pulls artifacts from remote artifact server(s) # # Args: - # targets (list of str): Targets to pull - # selection (_PipelineSelection): The selection mode for the specified targets - # ignore_junction_targets (bool): Whether junction targets should be filtered out - # remote (str): The URL of a specific remote server to pull from, or None + # targets: Targets to pull + # selection: The selection mode for the specified targets (_PipelineSelection) + # ignore_junction_targets: Whether junction targets should be filtered out + # artifact_remotes: Artifact cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # - # If `remote` specified as None, then regular configuration will be used - # to determine where to pull artifacts from. - # - def pull(self, targets, *, selection=_PipelineSelection.NONE, ignore_junction_targets=False, remote=None): + def pull( + self, + targets: Iterable[str], + *, + selection: str = _PipelineSelection.NONE, + ignore_junction_targets: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ): elements = self._load( targets, selection=selection, ignore_junction_targets=ignore_junction_targets, - connect_artifact_cache=True, - artifact_remote_url=remote, load_artifacts=True, attempt_artifact_metadata=True, + connect_artifact_cache=True, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) if not self._artifacts.has_fetch_remotes(): @@ -439,30 +517,38 @@ def pull(self, targets, *, selection=_PipelineSelection.NONE, ignore_junction_ta # push() # - # Pulls artifacts to remote artifact server(s) + # Pushes artifacts to remote artifact server(s), pulling them first if necessary, + # possibly from different remotes. # # Args: # targets (list of str): Targets to push # selection (_PipelineSelection): The selection mode for the specified targets # ignore_junction_targets (bool): Whether junction targets should be filtered out - # remote (str): The URL of a specific remote server to push to, or None - # - # If `remote` specified as None, then regular configuration will be used - # to determine where to push artifacts to. + # artifact_remotes: Artifact cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # # If any of the given targets are missing their expected buildtree artifact, # a pull queue will be created if user context and available remotes allow for # attempting to fetch them. # - def push(self, targets, *, selection=_PipelineSelection.NONE, ignore_junction_targets=False, remote=None): + def push( + self, + targets: Iterable[str], + *, + selection: str = _PipelineSelection.NONE, + ignore_junction_targets: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ): elements = self._load( targets, selection=selection, ignore_junction_targets=ignore_junction_targets, - connect_artifact_cache=True, - artifact_remote_url=remote, load_artifacts=True, + connect_artifact_cache=True, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) if not self._artifacts.has_push_remotes(): @@ -481,48 +567,54 @@ def push(self, targets, *, selection=_PipelineSelection.NONE, ignore_junction_ta # Checkout target artifact to the specified location # # Args: - # target (str): Target to checkout - # location (str): Location to checkout the artifact to - # force (bool): Whether files can be overwritten if necessary - # selection (_PipelineSelection): The selection mode for the specified targets - # integrate (bool): Whether to run integration commands - # hardlinks (bool): Whether checking out files hardlinked to - # their artifacts is acceptable - # tar (bool): If true, a tarball from the artifact contents will - # be created, otherwise the file tree of the artifact - # will be placed at the given location. If true and - # location is '-', the tarball will be dumped on the - # standard output. - # pull (bool): If true will attempt to pull any missing or incomplete - # artifacts. + # target: Target to checkout + # location: Location to checkout the artifact to + # force: Whether files can be overwritten if necessary + # selection: The selection mode for the specified targets (_PipelineSelection) + # integrate: Whether to run integration commands + # hardlinks: Whether checking out files hardlinked to + # their artifacts is acceptable + # tar: If true, a tarball from the artifact contents will + # be created, otherwise the file tree of the artifact + # will be placed at the given location. If true and + # location is '-', the tarball will be dumped on the + # standard output. + # pull: If true will attempt to pull any missing or incomplete + # artifacts. + # artifact_remotes: Artifact cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # def checkout( self, - target, + target: str, *, - location=None, - force=False, - selection=_PipelineSelection.RUN, - integrate=True, - hardlinks=False, - compression="", - pull=False, - tar=False, + location: Optional[str] = None, + force: bool = False, + selection: str = _PipelineSelection.RUN, + integrate: bool = True, + hardlinks: bool = False, + compression: str = "", + pull: bool = False, + tar: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, ): elements = self._load( (target,), selection=selection, - connect_artifact_cache=True, load_artifacts=True, attempt_artifact_metadata=True, + connect_artifact_cache=True, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) # self.targets contains a list of the loaded target objects # if we specify --deps build, Stream._load() will return a list # of build dependency objects, however, we need to prepare a sandbox # with the target (which has had its appropriate dependencies loaded) - target = self.targets[0] + element: Element = self.targets[0] self._check_location_writable(location, force=force, tar=tar) @@ -541,10 +633,10 @@ def checkout( _PipelineSelection.NONE: _Scope.NONE, _PipelineSelection.ALL: _Scope.ALL, } - with target._prepare_sandbox(scope=scope[selection], integrate=integrate) as sandbox: + with element._prepare_sandbox(scope=scope[selection], integrate=integrate) as sandbox: # Copy or move the sandbox to the target directory virdir = sandbox.get_virtual_directory() - self._export_artifact(tar, location, compression, target, hardlinks, virdir) + self._export_artifact(tar, location, compression, element, hardlinks, virdir) except BstError as e: raise StreamError( "Error while staging dependencies into a sandbox" ": '{}'".format(e), detail=e.detail, reason=e.reason @@ -702,31 +794,42 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE): # Checkout sources of the target element to the specified location # # Args: - # target (str): The target element whose sources to checkout - # location (str): Location to checkout the sources to - # force (bool): Whether to overwrite existing directories/tarfiles - # deps (str): The dependencies to checkout - # except_targets ([str]): List of targets to except from staging - # tar (bool): Whether to write a tarfile holding the checkout contents - # compression (str): The type of compression for tarball - # include_build_scripts (bool): Whether to include build scripts in the checkout + # target: The target element whose sources to checkout + # location: Location to checkout the sources to + # force: Whether to overwrite existing directories/tarfiles + # deps: The selection mode for the specified targets (_PipelineSelection) + # except_targets: List of targets to except from staging + # tar: Whether to write a tarfile holding the checkout contents + # compression: The type of compression for tarball + # include_build_scripts: Whether to include build scripts in the checkout + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # def source_checkout( self, - target, + target: str, *, - location=None, - force=False, - deps="none", - except_targets=(), - tar=False, - compression=None, - include_build_scripts=False, + location: Optional[str] = None, + force: bool = False, + deps=_PipelineSelection.NONE, + except_targets: Iterable[str] = (), + tar: bool = False, + compression: Optional[str] = None, + include_build_scripts: bool = False, + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_source_remotes: bool = False, ): self._check_location_writable(location, force=force, tar=tar) - elements = self._load((target,), selection=deps, except_targets=except_targets) + elements = self._load( + (target,), + selection=deps, + except_targets=except_targets, + connect_source_cache=True, + source_remotes=source_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) # Assert all sources are cached in the source dir self._fetch(elements) @@ -751,11 +854,28 @@ def source_checkout( # no_checkout (bool): Whether to skip checking out the source # force (bool): Whether to ignore contents in an existing directory # custom_dir (str): Custom location to create a workspace or false to use default location. + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # - def workspace_open(self, targets, *, no_checkout, force, custom_dir): + def workspace_open( + self, + targets: Iterable[str], + *, + no_checkout: bool = False, + force: bool = False, + custom_dir: Optional[str] = None, + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_source_remotes: bool = False, + ): # This function is a little funny but it is trying to be as atomic as possible. - elements = self._load(targets, selection=_PipelineSelection.REDIRECT) + elements = self._load( + targets, + selection=_PipelineSelection.REDIRECT, + connect_source_cache=True, + source_remotes=source_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) workspaces = self._context.get_workspaces() @@ -1198,8 +1318,8 @@ def _load_elements(self, target_groups): # def _load_elements_from_targets( self, - targets: List[str], - except_targets: List[str], + targets: Iterable[str], + except_targets: Iterable[str], *, rewritable: bool = False, valid_artifact_names: bool = False, @@ -1352,36 +1472,40 @@ def _track_cross_junction_filter(self, project, elements, cross_junction_request # fully loaded. # # Args: - # targets (list of str): Main targets to load - # selection (_PipelineSelection): The selection mode for the specified targets - # except_targets (list of str): Specified targets to except from fetching + # targets: Main targets to load + # selection: The selection mode for the specified targets (_PipelineSelection) + # except_targets: Specified targets to except from fetching # ignore_junction_targets (bool): Whether junction targets should be filtered out - # connect_artifact_cache (bool): Whether to try to contact remote artifact caches - # connect_source_cache (bool): Whether to try to contact remote source caches - # artifact_remote_url (str): A remote url for initializing the artifacts - # source_remote_url (str): A remote url for initializing source caches - # dynamic_plan (bool): Require artifacts as needed during the build - # load_artifacts (bool): Whether to load artifacts with artifact names - # attempt_artifact_metadata (bool): Whether to attempt to download artifact metadata in + # dynamic_plan: Require artifacts as needed during the build + # load_artifacts: Whether to load artifacts with artifact names + # attempt_artifact_metadata: Whether to attempt to download artifact metadata in # order to deduce build dependencies and reload. + # connect_artifact_cache: Whether to try to contact remote artifact caches + # connect_source_cache: Whether to try to contact remote source caches + # artifact_remotes: Artifact cache remotes specified on the commmand line + # source_remotes: Source cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects + # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # # Returns: # (list of Element): The primary element selection # def _load( self, - targets, + targets: Iterable[str], *, - selection=_PipelineSelection.NONE, - except_targets=(), - ignore_junction_targets=False, - connect_artifact_cache=False, - connect_source_cache=False, - artifact_remote_url=None, - source_remote_url=None, - dynamic_plan=False, - load_artifacts=False, - attempt_artifact_metadata=False, + selection: str = _PipelineSelection.NONE, + except_targets: Iterable[str] = (), + ignore_junction_targets: bool = False, + dynamic_plan: bool = False, + load_artifacts: bool = False, + attempt_artifact_metadata: bool = False, + connect_artifact_cache: bool = False, + connect_source_cache: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + source_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ignore_project_source_remotes: bool = False, ): elements, except_elements, artifacts = self._load_elements_from_targets( targets, except_targets, rewritable=False, valid_artifact_names=load_artifacts @@ -1390,7 +1514,7 @@ def _load( if artifacts: if selection in (_PipelineSelection.ALL, _PipelineSelection.RUN): raise StreamError( - "Error: '--deps {}' is not supported for artifact names".format(selection.value), + "Error: '--deps {}' is not supported for artifact names".format(selection), reason="deps-not-supported", ) @@ -1400,20 +1524,15 @@ def _load( # Hold on to the targets self.targets = elements - # FIXME: Instead of converting the URL to a RemoteSpec here, the CLI needs to - # be enhanced to parse a fully qualified RemoteSpec (including certs etc) - # from the command line, the CLI should be feeding the RemoteSpec through - # the Stream API directly. - # - artifact_remote = None - if artifact_remote_url: - artifact_remote = RemoteSpec(RemoteType.ALL, artifact_remote_url, push=True) - source_remote = None - if source_remote_url: - source_remote = RemoteSpec(RemoteType.ALL, source_remote_url, push=True) - # Connect to remote caches, this needs to be done before resolving element state - self._context.initialize_remotes(connect_artifact_cache, connect_source_cache, artifact_remote, source_remote) + self._context.initialize_remotes( + connect_artifact_cache, + connect_source_cache, + artifact_remotes, + source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) # In some cases we need to have an actualized artifact, with all of # it's metadata, such that we can derive attributes about the artifact @@ -1444,7 +1563,12 @@ def _load( # ensure those remotes are also initialized. # self._context.initialize_remotes( - connect_artifact_cache, connect_source_cache, artifact_remote, source_remote + connect_artifact_cache, + connect_source_cache, + artifact_remotes, + source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, ) self.targets += artifacts @@ -1806,7 +1930,7 @@ def _buildtree_pull_required(self, elements): # (list): artifact names present in the targets # def _expand_and_classify_targets( - self, targets: List[str], valid_artifact_names: bool = False + self, targets: Iterable[str], valid_artifact_names: bool = False ) -> Tuple[List[str], List[str]]: initial_targets = [] element_targets = [] diff --git a/src/buildstream/_utils.pyi b/src/buildstream/_utils.pyi index 458437281..1938eec08 100644 --- a/src/buildstream/_utils.pyi +++ b/src/buildstream/_utils.pyi @@ -1 +1,3 @@ def url_directory_name(url: str) -> str: ... +def valid_chars_name(name: str) -> bool: ... +def terminate_thread(thread_id: int) -> None: ... diff --git a/src/buildstream/_utils.pyx b/src/buildstream/_utils.pyx index 6605cc82a..2386ef430 100644 --- a/src/buildstream/_utils.pyx +++ b/src/buildstream/_utils.pyx @@ -22,6 +22,10 @@ This module contains utilities that have been optimized in Cython """ +from cpython.pystate cimport PyThreadState_SetAsyncExc +from cpython.ref cimport PyObject +from ._signals import TerminateException + def url_directory_name(str url): """Normalizes a url into a directory name @@ -35,6 +39,52 @@ def url_directory_name(str url): return ''.join([_transl(x) for x in url]) + +# terminate_thread() +# +# Ask a given a given thread to terminate by raising an exception in it. +# +# Args: +# thread_id (int): the thread id in which to throw the exception +# +def terminate_thread(long thread_id): + res = PyThreadState_SetAsyncExc(thread_id, TerminateException) + assert res == 1 + + +# Check if given filename containers valid characters. +# +# Args: +# name (str): Name of the file +# +# Returns: +# (bool): True if all characters are valid, False otherwise. +# +def valid_chars_name(str name): + cdef int char_value + cdef int forbidden_char + + for char_value in name: + # 0-31 are control chars, 127 is DEL, and >127 means non-ASCII + if char_value <= 31 or char_value >= 127: + return False + + # Disallow characters that are invalid on Windows. The list can be + # found at https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file + # + # Note that although : (colon) is not allowed, we do not raise + # warnings because of that, since we use it as a separator for + # junctioned elements. + # + # We also do not raise warnings on slashes since they are used as + # path separators. + for forbidden_char in '<>"|?*': + if char_value == forbidden_char: + return False + + return True + + ############################################################# # Module local helper Methods # ############################################################# diff --git a/src/buildstream/types.py b/src/buildstream/types.py index 67383f86a..4d9effc15 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -25,7 +25,8 @@ """ -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Union, Optional +import os from ._types import MetaFastEnum @@ -310,6 +311,25 @@ def __init__(self, project, provenance_node, duplicates, internal): self.internal = internal +# _HostMount() +# +# A simple object describing the behavior of a host mount. +# +class _HostMount: + def __init__(self, path: str, host_path: Optional[str] = None, optional: bool = False) -> None: + + # Support environment variable expansion in host mounts + path = os.path.expandvars(path) + if host_path is None: + host_path = path + else: + host_path = os.path.expandvars(host_path) + + self.path: str = path # Path inside the sandbox + self.host_path: str = host_path # Path on the host + self.optional: bool = optional # Optional mounts do not incur warnings or errors + + ######################################## # Type aliases # ######################################## diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index 21ef3becc..0632af098 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -128,7 +128,18 @@ def test_commands(cli, cmd, word_idx, expected): ("bst -", 1, MAIN_OPTIONS), ("bst --l", 1, ["--log-file "]), # Test that options of subcommands also complete - ("bst --no-colors build -", 3, ["--deps ", "-d ", "--remote ", "-r "]), + ( + "bst --no-colors build -", + 3, + [ + "--deps ", + "-d ", + "--artifact-remote ", + "--source-remote ", + "--ignore-project-artifact-remotes ", + "--ignore-project-source-remotes ", + ], + ), # Test the behavior of completing after an option that has a # parameter that cannot be completed, vs an option that has # no parameter diff --git a/tests/frontend/help.py b/tests/frontend/help.py index 20a93160f..f95cf02af 100644 --- a/tests/frontend/help.py +++ b/tests/frontend/help.py @@ -19,9 +19,7 @@ def test_help_main(cli): assert_help(result.output) -@pytest.mark.parametrize( - "command", [("artifact"), ("build"), ("checkout"), ("shell"), ("show"), ("source"), ("workspace")] -) +@pytest.mark.parametrize("command", [("artifact"), ("build"), ("shell"), ("show"), ("source"), ("workspace")]) def test_help(cli, command): result = cli.run(args=[command, "--help"]) result.assert_success() diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index 3e50b7261..70a700983 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -119,8 +119,8 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles): # Tests that: # -# * `bst artifact push --remote` pushes to the given remote, not one from the config -# * `bst artifact pull --remote` pulls from the given remote +# * `bst artifact push --artifact-remote` pushes to the given remote, not one from the config +# * `bst artifact pull --artifact-remote` pulls from the given remote # @pytest.mark.datafiles(DATA_DIR) def test_push_pull_specific_remote(cli, tmpdir, datafiles): @@ -142,7 +142,7 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles): cli.configure({"artifacts": {"servers": [{"url": bad_share.repo, "push": True},]}}) # Now try `bst artifact push` to the good_share. - result = cli.run(project=project, args=["artifact", "push", "target.bst", "--remote", good_share.repo]) + result = cli.run(project=project, args=["artifact", "push", "target.bst", "--artifact-remote", good_share.repo]) result.assert_success() # Assert that all the artifacts are in the share we pushed @@ -158,7 +158,7 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles): artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) - result = cli.run(project=project, args=["artifact", "pull", "target.bst", "--remote", good_share.repo]) + result = cli.run(project=project, args=["artifact", "pull", "target.bst", "--artifact-remote", good_share.repo]) result.assert_success() # And assert that it's again in the local cache, without having built @@ -417,7 +417,7 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): # Now check that a build with cli set as sharecli results in nothing being pulled, # as it doesn't have them cached and shareuser should be ignored. This # will however result in the artifacts being built and pushed to it - result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"]) + result = cli.run(project=project, args=["build", "--artifact-remote", sharecli.repo, "target.bst"]) result.assert_success() for element_name in all_elements: assert element_name not in result.get_pulled_elements() @@ -426,7 +426,7 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): # Now check that a clean build with cli set as sharecli should result in artifacts only # being pulled from it, as that was provided via the cli and is populated - result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"]) + result = cli.run(project=project, args=["build", "--artifact-remote", sharecli.repo, "target.bst"]) result.assert_success() for element_name in all_elements: assert cli.get_element_state(project, element_name) == "cached" diff --git a/tests/frontend/push.py b/tests/frontend/push.py index 4059b7c6f..a32c74701 100644 --- a/tests/frontend/push.py +++ b/tests/frontend/push.py @@ -529,7 +529,7 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir): # Use a separate local cache for this to ensure the complete element is pulled. cli2_path = os.path.join(str(tmpdir), "cli2") cli2 = Cli(cli2_path) - result = cli2.run(project=project, args=["artifact", "pull", "element1.bst", "--remote", share.repo]) + result = cli2.run(project=project, args=["artifact", "pull", "element1.bst", "--artifact-remote", share.repo]) result.assert_success() # Ensure element1 is cached locally @@ -594,7 +594,9 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) -def test_build_remote_option(caplog, cli, tmpdir, datafiles): +@pytest.mark.parametrize("use_remote", [True, False], ids=["with_cli_remote", "without_cli_remote"]) +@pytest.mark.parametrize("ignore_project", [True, False], ids=["ignore_project_caches", "include_project_caches"]) +def test_build_remote_option(caplog, cli, tmpdir, datafiles, use_remote, ignore_project): project = str(datafiles) caplog.set_level(1) @@ -609,16 +611,37 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): # Configure shareuser remote in user conf cli.configure({"artifacts": {"servers": [{"url": shareuser.repo, "push": True}]}}) - result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"]) + args = ["build", "target.bst"] + if use_remote: + args += ["--artifact-remote", sharecli.repo] + if ignore_project: + args += ["--ignore-project-artifact-remotes"] + + result = cli.run(project=project, args=args) # Artifacts should have only been pushed to sharecli, as that was provided via the cli result.assert_success() all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"] for element_name in all_elements: assert element_name in result.get_pushed_elements() - assert_shared(cli, sharecli, project, element_name) - assert_not_shared(cli, shareuser, project, element_name) - assert_not_shared(cli, shareproject, project, element_name) + + # Test shared state of project recommended cache depending + # on whether we decided to ignore project suggestions. + # + if ignore_project: + assert_not_shared(cli, shareproject, project, element_name) + else: + assert_shared(cli, shareproject, project, element_name) + + # If we specified a remote on the command line, this replaces any remotes + # specified in user configuration. + # + if use_remote: + assert_not_shared(cli, shareuser, project, element_name) + assert_shared(cli, sharecli, project, element_name) + else: + assert_shared(cli, shareuser, project, element_name) + assert_not_shared(cli, sharecli, project, element_name) # This test ensures that we are able to run `bst artifact push` in non strict mode diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py index 7af6ffcb7..86dac0bd8 100644 --- a/tests/sourcecache/fetch.py +++ b/tests/sourcecache/fetch.py @@ -91,7 +91,7 @@ def test_source_fetch(cli, tmpdir, datafiles): digest = sourcecache.export(source)._get_digest() # Push the source to the remote - res = cli.run(project=project_dir, args=["source", "push", "--remote", share.repo, element_name]) + res = cli.run(project=project_dir, args=["source", "push", "--source-remote", share.repo, element_name]) res.assert_success() # check the share has the proto and the object @@ -217,7 +217,7 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): digest = sourcecache.export(source)._get_digest() # Push the source to the remote - res = cli.run(project=project_dir, args=["source", "push", "--remote", share.repo, element_name]) + res = cli.run(project=project_dir, args=["source", "push", "--source-remote", share.repo, element_name]) res.assert_success() # Remove the cas content, only keep the proto and such around