Tristan/remote cli options

.pylintrc

@@ -5,10 +5,8 @@
 # run arbitrary code
 extension-pkg-whitelist=
     buildstream.node,
-    buildstream._loader._loader,
     buildstream._loader.loadelement,
     buildstream._loader.types,
-    buildstream._scheduler.jobs._job,
     buildstream._types,
     buildstream._utils,
     buildstream._variables,

NEWS

@@ -43,6 +43,15 @@ CLI
   o BREAKING CHANGE: `bst shell --use-buildtree` is now a boolean flag.
     `--use-buildtree=ask` and `--use-buildtree=try` are no longer supported.
 
+  o BREAKING CHANGE: `--remote` options removed, replaced by `--artifact-remote` or `--source-remote`
+
+  o BREAKING CHANGE: All old obsolete/deprecated commands removed, including:
+    - bst fetch (now bst source fetch)
+    - bst track (now bst source track)
+    - bst checkout (now bst artifact checkout)
+    - bst pull (now bst artifact pull)
+    - bst push (now bst artifact push)
+
 
 ==================
 buildstream 1.93.5

doc/source/using_commands.rst

@@ -9,6 +9,77 @@ invoked on the command line, where, in most cases, this will be from the
 project's main directory.
 
 
+Commonly used parameters
+------------------------
+
+
+.. _invoking_specify_remotes:
+
+Remotes
+~~~~~~~
+Remote :ref:`cache servers <config_cache_servers>` can be specified on the
+command line for commands which may result in communicating with such servers.
+
+Any command which has arguments to specify a ``REMOTE``, such as ``--artifact-remote``
+or ``--source-remote``, will override whatever was set in the user configuration,
+and will have an accompanying switch which allows the command to decide whether
+to ignore any remote :ref:`artifact <project_artifact_cache>` or :ref:`source <project_source_cache>`
+caches suggested by project configuration.
+
+Remotes can be specified on the command line either as a simple URI, or as
+a comma separated list of key value pairs.
+
+**Specifying a remote using a URI**
+
+.. code:: shell
+
+   bst artifact push --remote https://artifacts.com/artifacts:8088 element.bst
+
+**Specifying a remote using key value pairs**
+
+.. code:: shell
+
+   bst build --artifact-remote \
+       url=https://artifacts.com/artifacts:8088,type=index,server-cert=~/artifacts.cert \
+       element.bst
+
+
+Attributes
+''''''''''
+Here is the list attributes which can be spefied when providing a ``REMOTE`` on the command line:
+
+* ``url``
+
+  The URL of the remote, possibly including a port number.
+
+* ``instance-name``
+
+  The instance name of this remote, used for sharding by some implementations.
+
+* ``type``
+
+  Whether this remote is to be used for indexing, storage or both, as explained
+  in the corresponding :ref:`user configuration documentation <config_cache_servers>`
+
+* ``push``
+
+  Normally one need not specify this, as it is often inferred by the command
+  being used. In some cases, like :ref:`bst build <invoking_build>`, it can
+  be useful to specify multiple remotes, and only allow pushing to some of
+  the remotes.
+
+  If unspecified, this is assumed to be ``True`` and BuildStream will attempt
+  to push to the remote, but fallback to only pulling if insufficient credentials
+  were provided.
+
+* ``server-cert``, ``client-cert``, ``client-key``:
+
+  These keys specify the attributes of the :ref:`authentication configuration <config_remote_auth>`.
+
+  When specifying these on the command line, they are interpreted as paths relative
+  to the current working directory.
+
+
 Top-level commands
 ------------------
 

doc/source/using_config.rst

@@ -225,8 +225,14 @@ Using artifact :ref:`cache servers <config_cache_servers>` is an essential means
 *build avoidance*, as it will allow you to avoid building an element which has already
 been built and uploaded to a common artifact server.
 
-Artifact cache servers can be declared in three different ways, with differing
-priorities.
+Artifact cache servers can be declared in different ways, with differing priorities.
+
+
+Command line
+''''''''''''
+Various commands which involve connecting to artifact servers allow
+:ref:`specifying remotes <invoking_specify_remotes>`, remotes specified
+on the command line replace all user configuration.
 
 
 Global caches
@@ -300,8 +306,14 @@ to be rebuilt because of changes in the dependency graph, as BuildStream will fi
 to download the source code from the cache server before attempting to obtain it from an
 external source, which may suffer higher latencies.
 
-Source cache servers can be declared in three different ways, with differing
-priorities.
+Source cache servers can be declared in different ways, with differing priorities.
+
+
+Command line
+''''''''''''
+Various commands which involve connecting to source cache servers allow
+:ref:`specifying remotes <invoking_specify_remotes>`, remotes specified
+on the command line replace all user configuration.
 
 
 Global caches

setup.py

@@ -317,9 +317,7 @@ def files_from_module(modname):
 BUILD_EXTENSIONS = []
 
 register_cython_module("buildstream.node")
-register_cython_module("buildstream._loader._loader")
 register_cython_module("buildstream._loader.loadelement", dependencies=["buildstream.node"])
-register_cython_module("buildstream._scheduler.jobs._job")
 register_cython_module("buildstream._yaml", dependencies=["buildstream.node"])
 register_cython_module("buildstream._types")
 register_cython_module("buildstream._utils")

src/buildstream/_context.py

@@ -46,6 +46,26 @@
     # pylint: enable=cyclic-import
 
 
+# _CacheConfig
+#
+# A convenience object for parsing artifact/source cache configurations
+#
+class _CacheConfig:
+    def __init__(self, override_projects: bool, remote_specs: List[RemoteSpec]):
+        self.override_projects: bool = override_projects
+        self.remote_specs: List[RemoteSpec] = remote_specs
+
+    @classmethod
+    def new_from_node(cls, node: MappingNode) -> "_CacheConfig":
+        node.validate_keys(["override-project-caches", "servers"])
+        servers = node.get_sequence("servers", default=[], allowed_types=[MappingNode])
+
+        override_projects: bool = node.get_bool("push", default=False)
+        remote_specs: List[RemoteSpec] = [RemoteSpec.new_from_node(node) for node in servers]
+
+        return cls(override_projects, remote_specs)
+
+
 # Context()
 #
 # The Context object holds all of the user preferences
@@ -502,15 +522,19 @@ def get_toplevel_project(self) -> "Project":
     # Args:
     #    connect_artifact_cache: Whether to try to contact remote artifact caches
     #    connect_source_cache: Whether to try to contact remote source caches
-    #    artifact_remote: An overriding artifact cache remote, or None
-    #    source_remote: An overriding source cache remote, or None
+    #    artifact_remotes: Artifact cache remotes specified on the commmand line
+    #    source_remotes: Source cache remotes specified on the commmand line
+    #    ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
+    #    ignore_project_source_remotes: Whether to ignore artifact remotes specified by projects
     #
     def initialize_remotes(
         self,
         connect_artifact_cache: bool,
         connect_source_cache: bool,
-        artifact_remote: Optional[RemoteSpec],
-        source_remote: Optional[RemoteSpec],
+        artifact_remotes: Iterable[RemoteSpec] = (),
+        source_remotes: Iterable[RemoteSpec] = (),
+        ignore_project_artifact_remotes: bool = False,
+        ignore_project_source_remotes: bool = False,
     ) -> None:
 
         # Ensure all projects are fully loaded.
@@ -528,34 +552,6 @@ def initialize_remotes(
             if remote_execution:
                 self.pull_artifact_files, self.remote_execution_specs = self._load_remote_execution(remote_execution)
 
-        cli_artifact_remotes = [artifact_remote] if artifact_remote else []
-        cli_source_remotes = [source_remote] if source_remote else []
-
-        #
-        # Helper function to resolve which remote specs apply for a given project
-        #
-        def resolve_specs_for_project(
-            project: "Project", global_config: _CacheConfig, override_key: str, project_attribute: str,
-        ) -> List[RemoteSpec]:
-
-            # Obtain the overrides
-            override_node = self.get_overrides(project.name)
-            override_config_node = override_node.get_mapping(override_key, default={})
-            override_config = _CacheConfig.new_from_node(override_config_node)
-            if override_config.override_projects:
-                return override_config.remote_specs
-            elif global_config.override_projects:
-                return global_config.remote_specs
-
-            # If there were no explicit overrides, then take either the project specific
-            # config or fallback to the global config, and tack on the project recommended
-            # remotes at the end.
-            #
-            config_specs = override_config.remote_specs or global_config.remote_specs
-            project_specs = getattr(project, project_attribute)
-            all_specs = config_specs + project_specs
-            return list(utils._deduplicate(all_specs))
-
         #
         # Maintain our list of remote specs for artifact and source caches
         #
@@ -564,13 +560,22 @@ def resolve_specs_for_project(
             source_specs: List[RemoteSpec] = []
 
             if connect_artifact_cache:
-                artifact_specs = cli_artifact_remotes or resolve_specs_for_project(
-                    project, self._global_artifact_cache_config, "artifacts", "artifact_cache_specs",
+                artifact_specs = self._resolve_specs_for_project(
+                    project,
+                    artifact_remotes,
+                    ignore_project_artifact_remotes,
+                    self._global_artifact_cache_config,
+                    "artifacts",
+                    "artifact_cache_specs",
                 )
-
             if connect_source_cache:
-                source_specs = cli_source_remotes or resolve_specs_for_project(
-                    project, self._global_source_cache_config, "source-caches", "source_cache_specs",
+                source_specs = self._resolve_specs_for_project(
+                    project,
+                    source_remotes,
+                    ignore_project_source_remotes,
+                    self._global_source_cache_config,
+                    "source-caches",
+                    "source_cache_specs",
                 )
 
             # Advertize the per project remote specs publicly for the frontend
@@ -683,6 +688,64 @@ def get_cascache(self) -> CASCache:
     #                  Private methods                   #
     ######################################################
 
+    # _resolve_specs_for_project()
+    #
+    # Helper function to resolve which remote specs apply for a given project
+    #
+    # Args:
+    #    project: The project
+    #    cli_remotes: The remotes specified in the CLI
+    #    cli_override: Whether the CLI decided to override project suggestions
+    #    global_config: The global user configuration for this remote type
+    #    override_key: The key to lookup project overrides for this remote type
+    #    project_attribute: The Project attribute for project suggestions
+    #
+    # Returns:
+    #    The resolved remotes for this project.
+    #
+    def _resolve_specs_for_project(
+        self,
+        project: "Project",
+        cli_remotes: Iterable[RemoteSpec],
+        cli_override: bool,
+        global_config: _CacheConfig,
+        override_key: str,
+        project_attribute: str,
+    ) -> List[RemoteSpec]:
+
+        # Early return if the CLI is taking full control
+        if cli_override and cli_remotes:
+            return list(cli_remotes)
+
+        # Obtain the overrides
+        override_node = self.get_overrides(project.name)
+        override_config_node = override_node.get_mapping(override_key, default={})
+        override_config = _CacheConfig.new_from_node(override_config_node)
+
+        #
+        # Decide on what remotes to use from user config, if any
+        #
+        # Priority CLI -> Project overrides -> Global config
+        #
+        remotes: List[RemoteSpec]
+        if cli_remotes:
+            remotes = list(cli_remotes)
+        elif override_config.remote_specs:
+            remotes = override_config.remote_specs
+        else:
+            remotes = global_config.remote_specs
+
+        # If any of the configs have disabled project remotes, return now
+        #
+        if cli_override or override_config.override_projects or global_config.override_projects:
+            return remotes
+
+        # If there are any project recommendations, append them at the end
+        project_remotes = getattr(project, project_attribute)
+        remotes = list(utils._deduplicate(remotes + project_remotes))
+
+        return remotes
+
     # Force the resolved XDG variables into the environment,
     # this is so that they can be used directly to specify
     # preferred locations of things from user configuration
@@ -710,23 +773,3 @@ def _load_remote_execution(self, node: MappingNode) -> Tuple[bool, Optional[Remo
             remote_execution_specs = None
 
         return pull_artifact_files, remote_execution_specs
-
-
-# _CacheConfig
-#
-# A convenience object for parsing artifact/source cache configurations
-#
-class _CacheConfig:
-    def __init__(self, override_projects: bool, remote_specs: List[RemoteSpec]):
-        self.override_projects: bool = override_projects
-        self.remote_specs: List[RemoteSpec] = remote_specs
-
-    @classmethod
-    def new_from_node(cls, node: MappingNode) -> "_CacheConfig":
-        node.validate_keys(["override-project-caches", "servers"])
-        servers = node.get_sequence("servers", default=[], allowed_types=[MappingNode])
-
-        override_projects: bool = node.get_bool("push", default=False)
-        remote_specs: List[RemoteSpec] = [RemoteSpec.new_from_node(node) for node in servers]
-
-        return cls(override_projects, remote_specs)

src/buildstream/_frontend/app.py

@@ -711,7 +711,7 @@ def _handle_failure(self, element, task, failure):
                             _Scope.BUILD,
                             self.shell_prompt,
                             isolate=True,
-                            usebuildtree="always",
+                            usebuildtree=True,
                             unique_id=unique_id,
                         )
                     except BstError as e: