diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 81a489973..02f8c17f7 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -84,12 +84,40 @@ jobs: - checkout - setup_remote_docker: version: 20.10.11 + docker_layer_caching: true + - run: + name: Enable multiarch builds with QEMU + command: ./dev-scripts/enable-multiarch-docker - run: name: Build Debian package - command: ./dev-scripts/build-debian-pkg + # Building ARMv7 binaries is slow, so we only build them on the master + # branch. + command: | + set -exu + if [[ "${CIRCLE_BRANCH}" == 'master' ]]; then + readonly BUILD_TARGETS='linux/arm/v7,linux/amd64' + else + readonly BUILD_TARGETS='linux/amd64' + fi + ./dev-scripts/build-debian-pkg "${BUILD_TARGETS}" + - run: + name: Move Debian packages to a predictable location + # Building for multiple architectures at once, changes the path of the + # resulting Debian package. For example: + # - Building for both AMD64 and ARMv7, produces files in: + # - `releases/linux_arm_v7/*.deb` + # - `releases/linux_amd64/*.deb` + # - Building for only AMD64, produces files in: + # - `releases/*.deb` + # For consistency, we move all packages to `releases/*.deb`. + command: find . -name '*.deb' -exec mv {} ./debian-pkg/releases/ \; - run: name: Print Debian package contents - command: dpkg --contents debian-pkg/releases/tinypilot*.deb + command: | + set -exu + while read -r file; do + dpkg --contents "${file}" + done < <(ls ./debian-pkg/releases/*.deb) - persist_to_workspace: root: ./debian-pkg/releases paths: @@ -113,13 +141,16 @@ jobs: - run: name: Run lintian command: | - lintian \ - --check \ - --no-tag-display-limit \ - --suppress-tags-from-file .lintianignore \ - --no-cfg \ - --fail-on warning,error \ - tinypilot*.deb + set -exu + while read -r file; do + lintian \ + --check \ + --no-tag-display-limit \ + --suppress-tags-from-file .lintianignore \ + --no-cfg \ + --fail-on warning,error \ + "${file}" + done < <(ls *.deb) build_ansible_role: machine: image: ubuntu-2004:202010-01 @@ -142,7 +173,7 @@ jobs: - run: name: Add TinyPilot Debian package name as an environment variable command: | - TINYPILOT_DEBIAN_PACKAGE="$(ls ./debian-pkgs/tinypilot*.deb | xargs basename)" + TINYPILOT_DEBIAN_PACKAGE="$(ls ./debian-pkgs/tinypilot*amd64.deb | xargs basename)" echo "export TINYPILOT_DEBIAN_PACKAGE="${TINYPILOT_DEBIAN_PACKAGE}"" >> "${BASH_ENV}" - run: name: Create virtual environment @@ -258,6 +289,12 @@ workflows: - build_bundle: requires: - build_debian_package + # Creating the bundle assumes an ARMv7 build, so we only do this on + # master, because it's slow building the ARMv7 binaries from a + # CircleCI AMD64 instance. + filters: + branches: + only: master - verify_bundle: requires: - build_bundle diff --git a/.lintianignore b/.lintianignore index 64682aa83..9599917c4 100644 --- a/.lintianignore +++ b/.lintianignore @@ -1,3 +1,6 @@ +# These are rules for Lintian to ignore globally. For more targeted rules, see +# debian-pkg/debian/tinypilot.lintian-overrides. + # Debian doesn't want packages to install to /opt, but it also doesn't give # clear guidance on where they *should* go. It's too much churn at this point to # change, so we're going to ignore this. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c4f4c7dc7..d2ef10b9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,7 +22,8 @@ python3 -m venv venv && \ . venv/bin/activate && \ pip install --requirement requirements.txt && \ pip install --requirement dev_requirements.txt && \ - npm install + npm install && \ + ./dev-scripts/enable-multiarch-docker ``` ### Run automated tests diff --git a/ansible-role/defaults/main.yml b/ansible-role/defaults/main.yml index 000e20eaf..5720bdab6 100644 --- a/ansible-role/defaults/main.yml +++ b/ansible-role/defaults/main.yml @@ -12,5 +12,4 @@ tinypilot_port: 8000 tinypilot_keyboard_interface: /dev/hidg0 tinypilot_mouse_interface: /dev/hidg1 tinypilot_enable_debug_logging: no -tinypilot_pip_args: "" tinypilot_app_settings_file: "/home/{{ tinypilot_user }}/app_settings.cfg" diff --git a/ansible-role/tasks/main.yml b/ansible-role/tasks/main.yml index d3c602c21..f1cb1e4f4 100644 --- a/ansible-role/tasks/main.yml +++ b/ansible-role/tasks/main.yml @@ -8,24 +8,6 @@ apt: deb: "{{ tinypilot_debian_package_path }}" -- name: find absolute path to python3 - shell: realpath $(which python3) - register: realpath_python3 - changed_when: false - -- name: save absolute path to python3 - set_fact: - python3_abs_path: "{{ realpath_python3.stdout }}" - -- name: create TinyPilot virtualenv - pip: - virtualenv: "{{ tinypilot_dir }}/venv" - virtualenv_command: "{{ python3_abs_path }} -m venv venv" - requirements: "{{ tinypilot_dir }}/requirements.txt" - extra_args: "{{ tinypilot_pip_args }}" - notify: - - restart TinyPilot service - - name: create TinyPilot app settings template: src: tinypilot-app-settings.cfg.j2 diff --git a/bundler/create-bundle b/bundler/create-bundle index 57a1726c7..2fb4ce866 100755 --- a/bundler/create-bundle +++ b/bundler/create-bundle @@ -27,8 +27,11 @@ readonly OUTPUT_DIR='dist' readonly ANSIBLE_ROLES_DIR="${BUNDLE_DIR}/roles" readonly ANSIBLE_ROLE_TINYPILOT_DIR="${ANSIBLE_ROLES_DIR}/ansible-role-tinypilot" +# Exclude the AMD64 package from the production bundle. +rm -f "${BUNDLE_DIR}/tinypilot"*amd64.deb + # Ensure that a TinyPilot Debian package exists. -if ! ls "${BUNDLE_DIR}/tinypilot"*.deb 1> /dev/null 2>&1; then +if ! ls "${BUNDLE_DIR}/tinypilot"*armhf.deb 1> /dev/null 2>&1; then echo 'Failed to create bundle: no TinyPilot Debian package found.' >&2 exit 1 fi @@ -44,7 +47,7 @@ print_tinypilot_version() { dpkg-deb \ --show \ --showformat '${Tinypilot-Version}' \ - "${BUNDLE_DIR}/tinypilot"*.deb + "${BUNDLE_DIR}/tinypilot"*armhf.deb } # Compose bundle file name, which consists of these hyphen-separated parts: @@ -69,9 +72,14 @@ python3 -m venv venv pip install "pip>=21.3.1" pip install --requirement "${BUNDLE_DIR}/requirements.txt" -# Copy Ansible role. +# Clear Ansible roles from any previous bundle builds. +rm -rf "${ANSIBLE_ROLES_DIR}" mkdir "${ANSIBLE_ROLES_DIR}" -cp -r ../ansible-role "${ANSIBLE_ROLE_TINYPILOT_DIR}" + +# Copy Ansible role. +cp -r \ + --no-target-directory \ + ../ansible-role "${ANSIBLE_ROLE_TINYPILOT_DIR}" # Download Ansible role dependencies. ansible-galaxy install \ diff --git a/bundler/verify-bundle b/bundler/verify-bundle index bcff441d3..028050715 100755 --- a/bundler/verify-bundle +++ b/bundler/verify-bundle @@ -42,7 +42,7 @@ if [[ ! -f install ]]; then fi # List Debian package contents. -dpkg --contents tinypilot*.deb +dpkg --contents tinypilot*armhf.deb # Check that Ansible roles exist. readonly ANSIBLE_ROLES=( diff --git a/debian-pkg/Dockerfile b/debian-pkg/Dockerfile index c29a87f65..7e19f7b37 100644 --- a/debian-pkg/Dockerfile +++ b/debian-pkg/Dockerfile @@ -4,12 +4,17 @@ FROM debian:bullseye-20220328-slim AS build -RUN set -x && \ +RUN set -exu && \ apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ debhelper \ + dh-virtualenv \ dpkg-dev +# Docker populates this value from the --platform argument. See +# https://docs.docker.com/build/building/multi-platform/ +ARG TARGETPLATFORM + # The canonical, SemVer-compliant TinyPilot version. ARG TINYPILOT_VERSION @@ -20,14 +25,34 @@ ARG TINYPILOT_VERSION # recently built TinyPilot package. ARG PKG_VERSION -ARG PKG_NAME="tinypilot" -ARG PKG_BUILD_NUMBER="1" -ARG PKG_ARCH="all" -ARG PKG_ID="${PKG_NAME}-${PKG_VERSION}-${PKG_BUILD_NUMBER}-${PKG_ARCH}" - -RUN mkdir -p "/releases/${PKG_ID}" +ARG PKG_NAME='tinypilot' +ARG PKG_BUILD_NUMBER='1' + +# Docker's platform names don't match Debian's platform names, so we translate +# the platform name from the Docker version to the Debian version and save the +# result to a file so we can re-use it in later stages. +RUN cat | bash <<'EOF' +set -exu +case "${TARGETPLATFORM}" in + 'linux/amd64') + PKG_ARCH='amd64' + ;; + 'linux/arm/v7') + PKG_ARCH='armhf' + ;; + *) + echo "Unrecognized target platform: ${TARGETPLATFORM}" >&2 + exit 1 +esac +echo "${PKG_ARCH}" > /tmp/pkg-arch +echo "${PKG_NAME}-${PKG_VERSION}-${PKG_BUILD_NUMBER}-${PKG_ARCH}" > /tmp/pkg-id +EOF -WORKDIR "/releases/${PKG_ID}" +# We ultimately need the directory name to be the package ID, but there's no +# way to specify a dynamic value in Docker's WORKDIR command, so we use a +# placeholder directory name to assemble the Debian package and then rename the +# directory to its package ID name in the final stages of packaging. +WORKDIR /releases/placeholder-pkg-id COPY ./debian-pkg ./ COPY ./COPYRIGHT ./ @@ -39,9 +64,11 @@ COPY ./scripts ./scripts RUN echo "${TINYPILOT_VERSION}" > VERSION -WORKDIR "/releases/${PKG_ID}/debian" +WORKDIR /releases/placeholder-pkg-id/debian -RUN cat >control <control <= 11) Package: ${PKG_NAME} Architecture: ${PKG_ARCH} -Depends: adduser, python3, python3-pip, python3-venv, sudo +Depends: \${shlibs:Depends}, adduser, python3, python3-pip, python3-venv, sudo Homepage: https://tinypilotkvm.com Description: Simple, easy-to-use KVM over IP XBS-Tinypilot-Version: ${TINYPILOT_VERSION} EOF -RUN cat >changelog <changelog < $(date '+%a, %d %b %Y %H:%M:%S %z') EOF -WORKDIR "/releases/${PKG_ID}" -RUN dpkg-buildpackage --build=binary +# Rename the placeholder release directory to the final package ID. +WORKDIR /releases +RUN cat | bash <<'EOF' +set -exu +PKG_ID="$(cat /tmp/pkg-id)" +mv placeholder-pkg-id "${PKG_ID}" +cd "${PKG_ID}" +dpkg-buildpackage --build=binary +EOF FROM scratch as artifact -COPY --from=build "/releases/*.deb" ./ +COPY --from=build /releases/*.deb ./ diff --git a/debian-pkg/debian/rules b/debian-pkg/debian/rules index b1a2f2324..cb69017f4 100755 --- a/debian-pkg/debian/rules +++ b/debian-pkg/debian/rules @@ -1,8 +1,27 @@ #!/usr/bin/make -f +# Enable hardening build flags. +export DEB_BUILD_MAINT_OPTIONS = hardening=+all +DPKG_EXPORT_BUILDFLAGS = 1 +include /usr/share/dpkg/buildflags.mk + +export DH_VIRTUALENV_INSTALL_ROOT=/opt/tinypilot +# Prevent debhelper from generating an extra package with debug symbols. +export DEB_BUILD_OPTIONS=noddebs + %: - dh $@ + dh $@ --with python-virtualenv override_dh_installsystemd: dh_installsystemd --name=tinypilot-updater --no-start --no-enable dh_installsystemd --name=usb-gadget --no-start + +override_dh_virtualenv: + # Skip install because TinyPilot doesn't need to run setup.py to install. + # Use the venv directory because that's where we've historically kept it. + dh_virtualenv --skip-install --install-suffix venv + # dh_virtualenv doesn't remove __pycache__ directories, so we clean them up + # manually. + find . -type d -name __pycache__ -prune -exec rm -rf {} \; + # Lintian will complain if the .gitignore stays in venv. + rm ./debian/tinypilot$(DH_VIRTUALENV_INSTALL_ROOT)/venv/.gitignore diff --git a/debian-pkg/debian/tinypilot.lintian-overrides b/debian-pkg/debian/tinypilot.lintian-overrides new file mode 100644 index 000000000..67fbe93f7 --- /dev/null +++ b/debian-pkg/debian/tinypilot.lintian-overrides @@ -0,0 +1,17 @@ +# Suppress complaints about third-party dependencies we don't control. +tinypilot: embedded-javascript-library opt/tinypilot/venv/lib/python*/site-packages/werkzeug/debug/shared/jquery.js please use libjs-jquery +tinypilot: script-not-executable [opt/tinypilot/venv/lib/python*/site-packages/greenlet/tests/test_version.py] +tinypilot: script-not-executable [opt/tinypilot/venv/lib/python*/site-packages/pkg_resources/_vendor/appdirs.py] +tinypilot: script-not-executable [opt/tinypilot/venv/lib/python*/site-packages/setuptools/command/easy_install.py] +tinypilot: embedded-library libyaml opt/tinypilot/venv/lib/python*/site-packages/yaml/*.so +tinypilot: hardening-no-relro [opt/tinypilot/venv/lib/python*/site-packages/yaml/*.so] + +# Lintian doesn't recognize the Python interpreter when it's within the +# virtualenv. +tinypilot: unusual-interpreter /opt/tinypilot/venv/bin/python [opt/tinypilot/venv/bin/easy_install*] +tinypilot: unusual-interpreter /opt/tinypilot/venv/bin/python [opt/tinypilot/venv/bin/flask] +tinypilot: unusual-interpreter /opt/tinypilot/venv/bin/python [opt/tinypilot/venv/bin/pip*] +tinypilot: unusual-interpreter /opt/tinypilot/venv/bin/python [opt/tinypilot/venv/bin/wheel*] +tinypilot: unusual-interpreter python [opt/tinypilot/venv/lib/python*/site-packages/greenlet/tests/test_version.py] +tinypilot: unusual-interpreter python [opt/tinypilot/venv/lib/python*/site-packages/pkg_resources/_vendor/appdirs.py] +tinypilot: unusual-interpreter python [opt/tinypilot/venv/lib/python*/site-packages/setuptools/command/easy_install.py] diff --git a/debian-pkg/debian/tinypilot.triggers b/debian-pkg/debian/tinypilot.triggers new file mode 100644 index 000000000..a00b0c99d --- /dev/null +++ b/debian-pkg/debian/tinypilot.triggers @@ -0,0 +1,9 @@ +# These triggers are based on guidance from the dh-virtualenv documentation. +# https://dh-virtualenv.readthedocs.io/en/1.2.1/tutorial.html#step-2-set-up-packaging-for-your-project + +# Register interest in Python interpreter changes and don't make the Python +# package dependent on the virtualenv package processing (noawait). +interest-noawait /usr/bin/python3 + +# Also provide a symbolic trigger for all dh-virtualenv packages. +interest-await dh-virtualenv-interpreter-update diff --git a/dev-scripts/build-debian-pkg b/dev-scripts/build-debian-pkg index 649e8a4da..e98fd091b 100755 --- a/dev-scripts/build-debian-pkg +++ b/dev-scripts/build-debian-pkg @@ -1,6 +1,20 @@ #!/bin/bash -# Exit on first failure. +# Build TinyPilot Debian packages. +# +# Usage: +# build-debian-pkg [target architectures] +# +# target architecture: A comma-separated list of architectures that Docker +# accepts for its --platform argument. If omitted, defaults to +# "linux/arm/v7,linux/amd64". The only supported targets are linux/arm/v7 and +# linux/amd64. +# +# Examples +# build-debian-pkg "linux/arm/v7" +# build-debian-pkg "linux/arm/v7,linux/amd64" + +# Exit build script on first failure. set -e # Echo commands before executing them, by default to stderr. @@ -9,6 +23,8 @@ set -x # Exit on unset variable. set -u +BUILD_TARGETS="${1:-linux/arm/v7,linux/amd64}" + print_tinypilot_version() { # Format build hash suffix according to SemVer (`-ghhhhhhh` -> `+hhhhhhh`). git describe --tags --long | @@ -28,8 +44,9 @@ if [[ -n "${CI:-}" ]]; then fi readonly DOCKER_PROGRESS -DOCKER_BUILDKIT=1 docker build \ +DOCKER_BUILDKIT=1 docker buildx build \ --file debian-pkg/Dockerfile \ + --platform "${BUILD_TARGETS}" \ --build-arg TINYPILOT_VERSION="${TINYPILOT_VERSION}" \ --build-arg PKG_VERSION="${PKG_VERSION}" \ --target=artifact \ diff --git a/dev-scripts/enable-multiarch-docker b/dev-scripts/enable-multiarch-docker new file mode 100755 index 000000000..e39b20c98 --- /dev/null +++ b/dev-scripts/enable-multiarch-docker @@ -0,0 +1,33 @@ +#!/bin/bash + +# Configure Docker to support multiarch builds, allowing it to use QEMU to build +# images targeting different CPU architectures. + +# Exit script on first failure. +set -e + +# Echo commands before executing them, by default to stderr. +set -x + +# Exit on unset variable. +set -u + +# Enable multiarch builds with QEMU. +docker run \ + --rm \ + --privileged \ + multiarch/qemu-user-static \ + --reset \ + -p yes + +# Create multiarch build context. +docker context create builder + +# Create multiplatform builder. +docker buildx create builder \ + --name builder \ + --driver docker-container \ + --use + +# Ensure builder has booted. +docker buildx inspect --bootstrap diff --git a/dev-scripts/install-from-source b/dev-scripts/install-from-source index a246c13ca..fc1e0451c 100755 --- a/dev-scripts/install-from-source +++ b/dev-scripts/install-from-source @@ -22,7 +22,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" readonly SCRIPT_DIR cd "${SCRIPT_DIR}/.." -./dev-scripts/build-debian-pkg +./dev-scripts/build-debian-pkg "linux/arm/v7" # Shellcheck recommends find instead of ls for non-alphanumeric filenames, but # we don't expect non-alphanumeric filenames, and the find equivalent is more