From 81e312326250b0d378997fcd0c87d9e956dd9e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= Date: Tue, 26 Dec 2023 04:28:47 +0100 Subject: [PATCH] Rename tmpdir to target and always preserve it, dropping --preserve There is nothing temporary about our "tmpdir" - its sole purpose is to contain the final product of the bootstrap process. Thus, removing it at the end of bootstrap amounts to doing the entire process for nothing. To remedy this, --tmpdir is renamed --target, keeping the -t short form, and defaulting to "target" instead of "tmp" to make its purpose clearer. The --preserve option is removed, as the target is now always preserved. --- .github/workflows/bwrap.yml | 16 +++++----- .github/workflows/lint.yml | 2 +- lib/generator.py | 60 ++++++++++++++++++------------------ lib/{tmpdir.py => target.py} | 32 +++++++------------ rootfs.py | 60 +++++++++++++++++------------------- 5 files changed, 79 insertions(+), 91 deletions(-) rename lib/{tmpdir.py => target.py} (76%) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 776ea20eb..66cea846e 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -43,15 +43,15 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 with: name: packages - path: tmp/external/repo/** + path: target/external/repo/** - name: Tar pass1 image - run: tar -cf pass1_image.tar tmp + run: tar -cf pass1_image.tar target - name: Archive pass1_image uses: actions/upload-artifact@v3 with: @@ -96,15 +96,15 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 with: name: internal_packages_pass2 - path: tmp/external/repo/** + path: target/external/repo/** - name: Tar pass2 image - run: tar -cf pass2_image.tar tmp + run: tar -cf pass2_image.tar target - name: Archive pass2_image uses: actions/upload-artifact@v3 with: @@ -149,10 +149,10 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 - name: Archive created packages if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 with: name: packages - path: tmp/external/repo/** + path: target/external/repo/** diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e2000ab22..b8fada9d4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,7 +21,7 @@ jobs: - name: Install pylint run: sudo pip3 install pylint - name: pylint - run: pylint rootfs.py lib/utils.py lib/generator.py lib/tmpdir.py --disable=duplicate-code + run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py --disable=duplicate-code shellcheck: name: Lint shell files diff --git a/lib/generator.py b/lib/generator.py index 326101a4c..0f16e2fa3 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -29,26 +29,26 @@ def __init__(self, arch, external_sources, early_preseed, repo_path): self.external_sources = external_sources self.repo_path = repo_path self.source_manifest = self.get_source_manifest(not self.external_sources) - self.tmp_dir = None + self.target_dir = None self.external_dir = None - def reuse(self, tmpdir): + def reuse(self, target): """ Reuse a previously prepared bwrap environment for further stages. """ - self.tmp_dir = tmpdir.path - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.target_dir = target.path + self.external_dir = os.path.join(self.target_dir, 'external') self.distfiles() - def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0): + def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0): """ Prepare basic media of live-bootstrap. /steps -- contains steps to be built / -- contains seed to allow steps to be built, containing custom scripts and stage0-posix """ - self.tmp_dir = tmpdir.path - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.target_dir = target.path + self.external_dir = os.path.join(self.target_dir, 'external') # We use ext3 here; ext4 actually has a variety of extensions that # have been added with varying levels of recency @@ -59,33 +59,33 @@ def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_siz # argument matrix ... or we could just use ext3 instead which # is effectively universally the same if kernel_bootstrap: - init_path = os.path.join(self.tmp_dir, 'init') + init_path = os.path.join(self.target_dir, 'init') os.mkdir(init_path) - self.tmp_dir = init_path + self.target_dir = init_path if self.repo_path or self.external_sources: - tmpdir.add_disk("external", filesystem="ext3") - tmpdir.mount_disk("external", "external") + target.add_disk("external", filesystem="ext3") + target.mount_disk("external", "external") else: - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.external_dir = os.path.join(self.target_dir, 'external') elif using_kernel: - self.tmp_dir = os.path.join(self.tmp_dir, 'disk') - tmpdir.add_disk("disk", + self.target_dir = os.path.join(self.target_dir, 'disk') + target.add_disk("disk", filesystem="ext3", size=(target_size + "M") if target_size else "16G", bootable=True) - tmpdir.mount_disk("disk", "disk") - self.external_dir = os.path.join(self.tmp_dir, 'external') + target.mount_disk("disk", "disk") + self.external_dir = os.path.join(self.target_dir, 'external') os.makedirs(self.external_dir, exist_ok=True) if self.early_preseed: # Extract tar containing preseed with tarfile.open(self.early_preseed, "r") as seed: - seed.extractall(self.tmp_dir) + seed.extractall(self.target_dir) shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'), - os.path.join(self.tmp_dir, 'kaem.x86')) + os.path.join(self.target_dir, 'kaem.x86')) else: self.stage0_posix() self.seed() @@ -101,25 +101,25 @@ def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_siz shutil.copytree(self.repo_path, repo_dir) if kernel_bootstrap: - self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size) + self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size) if kernel_bootstrap and (self.external_sources or self.repo_path): - tmpdir.umount_disk('external') + target.umount_disk('external') elif using_kernel: - tmpdir.umount_disk('disk') + target.umount_disk('disk') def steps(self): """Copy in steps.""" self.get_packages() - shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps')) + shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps')) def stage0_posix(self): """Copy in all of the stage0-posix""" stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix') for entry in os.listdir(stage0_posix_base_dir): orig = os.path.join(stage0_posix_base_dir, entry) - target = os.path.join(self.tmp_dir, entry) + target = os.path.join(self.target_dir, entry) if os.path.isfile(orig): shutil.copy2(orig, target) else: @@ -128,14 +128,14 @@ def stage0_posix(self): arch = stage0_arch_map.get(self.arch, self.arch) kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init')) + shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init')) def seed(self): """Copy in extra seed files""" seed_dir = os.path.join(self.git_dir, 'seed') for entry in os.listdir(seed_dir): if os.path.isfile(os.path.join(seed_dir, entry)): - shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry)) + shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry)) @staticmethod def add_fiwix_files(file_list_path, dirpath): @@ -149,14 +149,14 @@ def add_fiwix_files(file_list_path, dirpath): def create_fiwix_file_list(self): """Create a list of files to populate Fiwix file system""" - file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', + file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1', 'files', 'fiwix-file-list.txt') - shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', + shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1', 'files', 'early-artifacts-needed-after-fiwix.txt'), file_list_path) save_cwd = os.getcwd() - os.chdir(self.tmp_dir) + os.chdir(self.target_dir) self.add_fiwix_files(file_list_path, 'steps') self.add_fiwix_files(file_list_path, 'distfiles') os.chdir(save_cwd) @@ -170,7 +170,7 @@ def copy_no_network_distfiles(out): shutil.copy2(os.path.join(self.distfiles_dir, file), os.path.join(out, file)) - early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles') + early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles') main_distfile_dir = os.path.join(self.external_dir, 'distfiles') if early_distfile_dir != main_distfile_dir: @@ -217,7 +217,7 @@ def append_srcfs(self, image_file): """Append srcfs file system to disk image""" save_cwd = os.getcwd() - os.chdir(self.tmp_dir) + os.chdir(self.target_dir) self.output_tree(image_file, '.') # Add commands to kick off stage0-posix diff --git a/lib/tmpdir.py b/lib/target.py similarity index 76% rename from lib/tmpdir.py rename to lib/target.py index 954235ccc..db4b47467 100644 --- a/lib/tmpdir.py +++ b/lib/target.py @@ -4,34 +4,32 @@ # SPDX-License-Identifier: GPL-3.0-or-later """ -Contains a class that represents a tmpdir +Contains a class that represents a target directory """ import enum import getpass import os -import shutil from lib.utils import mount, umount, create_disk, run_as_root -class TmpType(enum.Enum): - """Different types of tmpdirs we can have""" +class TargetType(enum.Enum): + """Different types of target dirs we can have""" NONE = 0 TMPFS = 1 -class Tmpdir: +class Target: """ - Represents a tmpdir + Represents a target directory """ _disks = {} _disk_filesystems = {} _mountpoints = {} - def __init__(self, preserve, path="tmp"): + def __init__(self, path="target"): self.path = os.path.abspath(path) - self.preserve = preserve - self._type = TmpType.NONE + self._type = TargetType.NONE if not os.path.exists(self.path): os.mkdir(self.path) @@ -41,23 +39,15 @@ def __del__(self): print(f"Unmounting {path}") umount(path) - if not self.preserve: - for disk in self._disks.values(): - print(f"Detaching {disk}") - run_as_root("losetup", "-d", disk) - - if self._type == TmpType.TMPFS: - print(f"Unmounting tmpdir from {self.path}") - umount(self.path) - - print(f"Removing {self.path}") - shutil.rmtree(self.path, ignore_errors=True) + for disk in self._disks.values(): + print(f"Detaching {disk}") + run_as_root("losetup", "-d", disk) def tmpfs(self, size="8G"): """Mount a tmpfs""" print(f"Mounting tmpfs on {self.path}") mount("tmpfs", self.path, "tmpfs", f"size={size}") - self._type = TmpType.TMPFS + self._type = TargetType.TMPFS # pylint: disable=too-many-arguments def add_disk(self, diff --git a/rootfs.py b/rootfs.py index 18834ab71..a08d27dcd 100755 --- a/rootfs.py +++ b/rootfs.py @@ -17,7 +17,7 @@ import os from lib.utils import run, run_as_root -from lib.tmpdir import Tmpdir +from lib.target import Target from lib.generator import Generator, stage0_arch_map def create_configuration_file(args): @@ -58,11 +58,9 @@ def main(): action="store_true") parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox", action="store_true") - parser.add_argument("-p", "--preserve", help="Do not remove temporary dir", - action="store_true") - parser.add_argument("-t", "--tmpdir", help="Temporary directory", - default="tmp") - parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir", + parser.add_argument("-t", "--target", help="Target directory", + default="target") + parser.add_argument("--tmpfs", help="Use a tmpfs on target", action="store_true") parser.add_argument("--tmpfs-size", help="Size of the tmpfs", default="8G") @@ -128,7 +126,7 @@ def check_types(): if args.arch != "x86": print("Only x86 is supported at the moment, other arches are for development only.") - # Tmp validation + # Tmpfs validation if args.bwrap and args.tmpfs: raise ValueError("tmpfs cannot be used with bwrap.") @@ -156,19 +154,19 @@ def check_types(): with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'): pass - # tmpdir - tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve) + # target + target = Target(path=args.target) if args.tmpfs: - tmpdir.tmpfs(size=args.tmpfs_size) + target.tmpfs(size=args.tmpfs_size) generator = Generator(arch=args.arch, external_sources=args.external_sources, repo_path=args.repo, early_preseed=args.early_preseed) - bootstrap(args, generator, tmpdir, args.target_size) + bootstrap(args, generator, target, args.target_size) -def bootstrap(args, generator, tmpdir, size): +def bootstrap(args, generator, target, size): """Kick off bootstrap process.""" print(f"Bootstrapping {args.arch}") if args.chroot: @@ -179,28 +177,28 @@ def bootstrap(args, generator, tmpdir, size): chroot_binary = run_as_root('python3', '-c', find_chroot, capture_output=True).stdout.decode().strip() - generator.prepare(tmpdir, using_kernel=False) + generator.prepare(target, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init) + run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init) elif args.bwrap: init = '/init' if not args.internal_ci or args.internal_ci == "pass1": - generator.prepare(tmpdir, using_kernel=False) + generator.prepare(target, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') else: - generator.reuse(tmpdir) + generator.reuse(target) run('env', '-i', 'bwrap', '--unshare-user', '--uid', '0', '--gid', '0', '--unshare-net' if args.external_sources else None, '--setenv', 'PATH', '/usr/bin', - '--bind', generator.tmp_dir, '/', + '--bind', generator.target_dir, '/', '--dir', '/dev', '--dev-bind', '/dev/null', '/dev/null', '--dev-bind', '/dev/zero', '/dev/zero', @@ -216,44 +214,44 @@ def bootstrap(args, generator, tmpdir, size): elif args.bare_metal: if args.kernel: - generator.prepare(tmpdir, using_kernel=True, target_size=size) - image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) + generator.prepare(target, using_kernel=True, target_size=size) + path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target)) print("Please:") - print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.") - print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.") + print(f" 1. Take {path}/initramfs and your kernel, boot using this.") + print(f" 2. Take {path}/disk.img and put this on a writable storage medium.") else: - generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) - image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) + generator.prepare(target, kernel_bootstrap=True, target_size=size) + path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target)) print("Please:") - print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.") + print(f" 1. Take {path}.img and write it to a boot drive and then boot it.") else: if args.kernel: - generator.prepare(tmpdir, using_kernel=True, target_size=size) + generator.prepare(target, using_kernel=True, target_size=size) run(args.qemu_cmd, '-enable-kvm', '-m', str(args.qemu_ram) + 'M', '-smp', str(args.cores), '-no-reboot', - '-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw', - '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', + '-drive', 'file=' + target.get_disk("disk") + ',format=raw', + '-drive', 'file=' + target.get_disk("external") + ',format=raw', '-nic', 'user,ipv6=off,model=e1000', '-kernel', args.kernel, '-nographic', '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw') else: - generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) + generator.prepare(target, kernel_bootstrap=True, target_size=size) arg_list = [ '-enable-kvm', '-m', str(args.qemu_ram) + 'M', '-smp', str(args.cores), '-no-reboot', - '-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw' + '-drive', 'file=' + generator.target_dir + '.img' + ',format=raw' ] - if tmpdir.get_disk("external") is not None: + if target.get_disk("external") is not None: arg_list += [ - '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', + '-drive', 'file=' + target.get_disk("external") + ',format=raw', ] arg_list += [ '-machine', 'kernel-irqchip=split',