Rework kci docker with new framework

kci

@@ -18,6 +18,9 @@ import json
 import click
 
 from kernelci.cli import Args, kci
+from kernelci.cli import (  # pylint: disable=unused-import
+    docker as kci_docker,
+)
 import kernelci.api
 import kernelci.config
 

kernelci/cli/docker.py

@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+# Copyright (C) 2022, 2023 Collabora Limited
+# Author: Guillaume Tucker <guillaume.tucker@collabora.com>
+
+"""Tool to build and manage Docker images"""
+
+import functools
+
+import click
+
+from kernelci.docker import Docker
+from . import Args, kci
+
+
+def kci_docker_args(func):
+    """Decorator to add common arguments to a `kci docker` command"""
+    @click.argument('image')
+    @click.argument('fragments', nargs=-1)
+    @click.option(
+        '--arch',
+        help="CPU architecture, e.g. x86"
+    )
+    @click.option(
+        '--prefix',
+        help="Prefix used in Docker image names and tags"
+    )
+    @click.option(
+        '--version',
+        help="Image version tag, e.g. 20221011.0"
+    )
+    @functools.wraps(func)
+    def new_func(*args, **kwargs):
+        return func(*args, **kwargs)
+    return new_func
+
+
+@kci.group(name='docker')
+def kci_docker():
+    """Build and manage Docker images
+
+    The image and fragment names are positional arguments.  For example:
+
+        kci docker build gcc-10 --arch=x86 kernelci kselftest
+
+        kci docker generate kernelci
+    """
+
+
+@kci_docker.command
+@kci_docker_args
+def generate(image, fragments, arch, prefix, version):
+    """Generate a Dockerfile"""
+    helper = Docker(image, fragments, arch, prefix, version)
+    click.echo(helper.get_dockerfile())
+
+
+@kci_docker.command
+@kci_docker_args
+def name(image, fragments, arch, prefix, version):
+    """Get the full name of a Docker image without building it"""
+    helper = Docker(image, fragments, arch, prefix, version)
+    _, _, image_name = helper.get_image_name()
+    click.echo(image_name)
+
+
+def _get_docker_args(build_arg):
+    try:
+        docker_args = dict(
+            barg.split('=') for barg in build_arg
+        ) if build_arg else {}
+        return docker_args
+    except ValueError as exc:
+        raise click.UsageError(f"Invalid --build-arg value: {exc}")
+
+
+def _do_push(helper, base_name, tag_name, verbose):
+    push_log = helper.push_image(base_name, tag_name)
+    if verbose:
+        for line in helper.iterate_push_log(push_log):
+            click.echo(line)
+        for line in push_log:
+            error = line.get('errorDetail')
+            if error:
+                raise click.ClickException(error['message'])
+
+
+@kci_docker.command
+@kci_docker_args
+@Args.verbose
+@click.option('--build-arg', multiple=True, help="Docker build arguments")
+@click.option('--cache/--no-cache', default=True, help="Use docker cache")
+@click.option('--push/--no-push', help="Push the image to Docker Hub")
+def build(image, fragments, arch, prefix, **kwargs):
+    """Build a Docker image"""
+    helper = Docker(image, fragments, arch, prefix, kwargs.get('version'))
+    base_name, tag_name, img_name = helper.get_image_name()
+    click.echo(f"Building {img_name}")
+    verbose = kwargs.get('verbose')
+    if verbose:
+        click.echo(helper.get_dockerfile())
+    docker_args = _get_docker_args(kwargs.get('build_arg'))
+    nocache = not kwargs.get('cache')
+    build_log, build_err = helper.build_image(img_name, docker_args, nocache)
+    if verbose:
+        for line in helper.iterate_build_log(build_log):
+            click.echo(line)
+    if build_err:
+        raise click.ClickException(build_err)
+    if kwargs.get('push'):
+        click.echo(f"Pushing {img_name} to Docker Hub")
+        _do_push(helper, base_name, tag_name, verbose)

kernelci/docker.py

@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+# Copyright (C) 2022, 2023 Collabora Limited
+# Author: Guillaume Tucker <guillaume.tucker@collabora.com>
+
+"""Generate Dockerfiles from Jinja2 templates and build images"""
+
+import io
+import json
+
+import docker
+from jinja2 import Environment, FileSystemLoader
+
+
+class Docker:
+    """Helper class to manage KernelCI Docker images"""
+
+    TEMPLATE_PATHS = [
+        'config/docker',
+        '/etc/kernelci/config/docker',
+    ]
+
+    def __init__(self, image, fragments, arch, prefix, version):
+        self._image = image
+        self._fragments = fragments or []
+        self._arch = arch or ''
+        self._prefix = prefix or ''
+        self._version = version or ''
+
+    @classmethod
+    def iterate_build_log(cls, build_log):
+        """Iterate all the lines in a Docker build log"""
+        for chunk in build_log:
+            stream = chunk.get('stream') or ""
+            for line in stream.splitlines():
+                yield line
+
+    @classmethod
+    def iterate_push_log(cls, push_log):
+        """Iterate all the status lines in a Docker push log"""
+        for line in push_log:
+            if 'status' in line and 'progressDetail' not in line:
+                yield line['status']
+
+    @classmethod
+    def push_image(cls, base_name, tag_name):
+        """Push a Docker image to Docker Hub"""
+        client = docker.from_env()
+        push_log_json = client.images.push(base_name, tag_name)
+        return list(
+            json.loads(json_line)
+            for json_line in push_log_json.splitlines()
+        )
+
+    def get_image_name(self):
+        """Get the base name, tag name and full image name"""
+        base_name = self._prefix + self._image
+        tag_strings = [self._arch] if self._arch else []
+        if self._fragments:
+            tag_strings.extend(self._fragments)
+        if self._version:
+            tag_strings.append(self._version)
+        tag_name = '-'.join(tag_strings)
+        name = ':'.join((base_name, tag_name)) if tag_name else base_name
+        return base_name, tag_name, name
+
+    def get_dockerfile(self):
+        """Get the generated Dockerfile"""
+        params = {
+            'prefix': self._prefix,
+            'fragments': [
+                f'fragment/{fragment}.jinja2'
+                for fragment in self._fragments
+            ] if self._fragments else []
+        }
+        template_name = (
+            '-'.join((self._image, self._arch)) if self._arch else self._image
+        )
+        jinja2_env = Environment(loader=FileSystemLoader(self.TEMPLATE_PATHS))
+        template = jinja2_env.get_template(f"{template_name}.jinja2")
+        return template.render(params)
+
+    def build_image(self, name, buildargs, nocache, dockerfile=None):
+        """Build the Docker image"""
+        if dockerfile is None:
+            dockerfile = self.get_dockerfile()
+        client = docker.from_env()
+        dockerfile_obj = io.BytesIO(dockerfile.encode())
+        try:
+            _, build_log = client.images.build(
+                fileobj=dockerfile_obj, tag=name, buildargs=buildargs,
+                nocache=nocache
+            )
+            build_err = None
+        except docker.errors.BuildError as exc:
+            build_log = exc.build_log
+            build_err = exc.msg
+        return build_log, build_err

kernelci/legacy/cli/__init__.py

@@ -12,7 +12,6 @@
 from . import (
     api,
     config,
-    docker,
     job,
     node,
     pubsub,
@@ -25,7 +24,6 @@
 _COMMANDS = {
     'api': api.main,
     'config': config.main,
-    'docker': docker.main,
     'job': job.main,
     'node': node.main,
     'pubsub': pubsub.main,