From 734403e1ca766162b36b1abe9b0849a2683ce0e5 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 24 Mar 2026 17:07:03 -0700 Subject: [PATCH] continuous-integration2: Switch to ruff for Python formatting We started using ruff for linting a while ago. Since then, it has grown a formatting option, which is quite fast. While it is a little less flexible than YAPF, we don't currently customize the behavior aside from some exemptions. Switch to ruff for formatting. Most of the changes end up making the code a little more readable, at the expense of some extra lines. Switch to the generic fmt comments instead of the yapf specific ones. Use the option to preserve our quoting style, which uses single quotes for string literals and double quotes for f-string or raw strings. Signed-off-by: Nathan Chancellor --- caching/check.py | 54 +++++++++++-------- caching/update.py | 34 ++++++------ generator/generate.py | 42 ++++++++------- generator/generate_tuxsuite.py | 62 ++++++++++----------- generator/generate_workflow.py | 75 ++++++++++++++------------ ruff.toml | 3 ++ scripts/build-local.py | 83 ++++++++++++++--------------- scripts/check-logs.py | 37 +++++++++---- scripts/check-patches-apply.py | 57 ++++++++++---------- scripts/check-patches.py | 11 ++-- scripts/estimate-builds.py | 17 +++--- scripts/generate-boot-utils-json.py | 17 ++++-- scripts/markdown-badges.py | 31 ++++++----- scripts/parse-debian-clang.py | 30 ++++++----- scripts/visualize-builds.py | 39 +++++++------- utils.py | 49 ++++++++--------- 16 files changed, 347 insertions(+), 294 deletions(-) diff --git a/caching/check.py b/caching/check.py index 3b106740..7ac893a9 100755 --- a/caching/check.py +++ b/caching/check.py @@ -36,6 +36,7 @@ we want to only cache on a "pass" or "fail" status as these mean that Tuxsuite actually completed its work and didnt timeout. """ + import argparse import json import os @@ -48,7 +49,11 @@ # pylint: disable-next=import-error import requests -from utils import get_patches_hash, get_workflow_name_to_var_name, update_repository_variable +from utils import ( + get_patches_hash, + get_workflow_name_to_var_name, + update_repository_variable, +) OWNER = "ClangBuiltLinux" REPO = "continuous-integration2" @@ -63,8 +68,7 @@ TIMEOUT = 64 -class MalformedCacheError(Exception): - ... +class MalformedCacheError(Exception): ... def parse_args(): @@ -107,7 +111,8 @@ def ___purge___cache___(): list_response = requests.get(list_url, headers=headers, timeout=TIMEOUT) print(list_response.content) all_variables_keys = [ - x["name"] for x in json.loads(list_response.content)["variables"] + x["name"] + for x in json.loads(list_response.content)["variables"] if x["name"].startswith("_") ] @@ -115,9 +120,7 @@ def ___purge___cache___(): delete_url = ( f"https://api.github.com/repos/{OWNER}/{REPO}/actions/variables/{key}" ) - delete_response = requests.delete(delete_url, - headers=headers, - timeout=TIMEOUT) + delete_response = requests.delete(delete_url, headers=headers, timeout=TIMEOUT) if delete_response.status_code != 204: print(f"ERROR: Couldn't delete cache entry with key {key}") sys.exit(1) @@ -154,16 +157,19 @@ def get_repository_variable_or_none(name: str) -> Optional[dict]: return json.loads(as_dict["value"]) -def create_repository_variable(name: str, linux_sha: str, clang_version: str, - patches_hash: str) -> None: +def create_repository_variable( + name: str, linux_sha: str, clang_version: str, patches_hash: str +) -> None: _url = f"https://api.github.com/repos/{OWNER}/{REPO}/actions/variables" - _value = json.dumps({ - "linux_sha": linux_sha, - "clang_version": clang_version, - "patches_hash": patches_hash, - "build_status": "presuite", - }) + _value = json.dumps( + { + "linux_sha": linux_sha, + "clang_version": clang_version, + "patches_hash": patches_hash, + "build_status": "presuite", + } + ) data = {"name": name, "value": _value} resp = requests.post(_url, headers=headers, json=data, timeout=TIMEOUT) @@ -199,8 +205,10 @@ def create_repository_variable(name: str, linux_sha: str, clang_version: str, # pull down repo variable result = get_repository_variable_or_none(VAR_NAME) if result is None: - print(f"CACHE MISS: Did not find repo variable {VAR_NAME} " - f"from workflow_name: {args.workflow_name}. Creating it now.") + print( + f"CACHE MISS: Did not find repo variable {VAR_NAME} " + f"from workflow_name: {args.workflow_name}. Creating it now." + ) create_repository_variable( VAR_NAME, linux_sha=curr_sha, @@ -219,14 +227,19 @@ def create_repository_variable(name: str, linux_sha: str, clang_version: str, raise MalformedCacheError( f"The cache with key {VAR_NAME} based on workflow '{args.workflow_name}' " f"is one or more fields. It's missing: {missing_fields}\n" - f"The current cache looks as follows:\n{result}.") + f"The current cache looks as follows:\n{result}." + ) cached_sha = result["linux_sha"] cached_clang_version = result["clang_version"] cached_build_status = result["build_status"] cached_patches_hash = result.get("patches_hash", curr_patches_hash) - if cached_sha != curr_sha or cached_clang_version != curr_clang_version or cached_patches_hash != curr_patches_hash: + if ( + cached_sha != curr_sha + or cached_clang_version != curr_clang_version + or cached_patches_hash != curr_patches_hash + ): print(f"""\ CACHE MISS: current linux_sha is {curr_sha}, clang_version is {curr_clang_version}, and current patches_hash is {curr_patches_hash} while {args.workflow_name} has @@ -271,5 +284,4 @@ def create_repository_variable(name: str, linux_sha: str, clang_version: str, with open(env_file, "a", encoding="utf-8") as fd: fd.write(f"CACHE_PASS={cached_build_status.strip()}") - sys.exit( - 0) # signifies to the workflow that no jobs should run ('success') + sys.exit(0) # signifies to the workflow that no jobs should run ('success') diff --git a/caching/update.py b/caching/update.py index 46975c27..33bfb1c0 100755 --- a/caching/update.py +++ b/caching/update.py @@ -29,7 +29,11 @@ import urllib.request from pathlib import Path -from utils import get_patches_hash, get_workflow_name_to_var_name, update_repository_variable +from utils import ( + get_patches_hash, + get_workflow_name_to_var_name, + update_repository_variable, +) if "GITHUB_WORKFLOW" not in os.environ: print("Couldn't find GITHUB_WORKFLOW in env. Not in a GitHub Workflow?") @@ -38,15 +42,12 @@ MOCK = "MOCK" in os.environ -def update_cache(status: str, git_sha: str, clang_version: str, - patches_hash: str): +def update_cache(status: str, git_sha: str, clang_version: str, patches_hash: str): print(f"Trying to update cache with status: {status}") - cache_entry_key = get_workflow_name_to_var_name( - os.environ["GITHUB_WORKFLOW"]) + cache_entry_key = get_workflow_name_to_var_name(os.environ["GITHUB_WORKFLOW"]) if "REPO_SCOPED_PAT" not in os.environ: - print( - "Couldn't find REPO_SCOPED_PAT in env. Not in a GitHub Workflow?") + print("Couldn't find REPO_SCOPED_PAT in env. Not in a GitHub Workflow?") sys.exit(1) headers = {"Authorization": f"Bearer {os.environ['REPO_SCOPED_PAT']}"} @@ -82,8 +83,7 @@ def main(): for entry, build in builds.items(): try: git_sha = build["git_sha"] - clang_version = build["tuxmake_metadata"]["compiler"][ - "version_full"] + clang_version = build["tuxmake_metadata"]["compiler"]["version_full"] break except KeyError: builds_that_are_missing_metadata.append(entry) @@ -100,32 +100,32 @@ def main(): build_log_raw = response.read().decode() failed_pattern = ( - r"(?<=Apply patch set FAILED\s)[0-9A-Za-z._:/\-\s]*?(?=\serror: )") + r"(?<=Apply patch set FAILED\s)[0-9A-Za-z._:/\-\s]*?(?=\serror: )" + ) failed_matches = re.findall(failed_pattern, build_log_raw) if len(failed_matches) == 0: print( f"No patches failed to apply yet the build status stated there were: {build['status_message']}" ) - sys.exit( - 0) # Not sure how we got here but continue the action anyways + sys.exit(0) # Not sure how we got here but continue the action anyways patches_that_failed_to_apply = failed_matches[0].split('\n') - print( - f"Error: Some patches failed to apply.\n{patches_that_failed_to_apply}\n" - ) + print(f"Error: Some patches failed to apply.\n{patches_that_failed_to_apply}\n") sys.exit(1) if len(builds_that_are_missing_metadata) == len(builds): raise RuntimeError( f"Could not find a suitable git sha or compiler version in any build\n" - f"Here's the build.json:\n{raw}") + f"Here's the build.json:\n{raw}" + ) if len(builds_that_are_missing_metadata) > 0: print( "Warning: Some of the builds in builds.json are malformed and missing " "some metadata.\n" f"Here's a list: {builds_that_are_missing_metadata}\n" - f"Here's the build.json in question:\n{raw}") + f"Here's the build.json in question:\n{raw}" + ) assert git_sha and clang_version diff --git a/generator/generate.py b/generator/generate.py index e28fb965..b306f88c 100755 --- a/generator/generate.py +++ b/generator/generate.py @@ -17,20 +17,22 @@ def parse_args(trees): - parser = ArgumentParser( - description='Generate yml files and perform extra checks') + parser = ArgumentParser(description='Generate yml files and perform extra checks') - parser.add_argument('-c', - '--check', - action='store_true', - help='Fail if generating yml files results in a diff') + parser.add_argument( + '-c', + '--check', + action='store_true', + help='Fail if generating yml files results in a diff', + ) parser.add_argument( 'trees', choices=[*trees, 'all'], default='all', help='The trees to generate yml files for (default: all)', metavar='TREES', - nargs='*') + nargs='*', + ) return parser.parse_args() @@ -38,13 +40,13 @@ def parse_args(trees): def update_llvm_tot_version(): # Avoids pulling in an extra Python package dependency curl_cmd = [ - 'curl', '-fLSs', - 'https://raw.githubusercontent.com/llvm/llvm-project/main/cmake/Modules/LLVMVersion.cmake' + 'curl', + '-fLSs', + 'https://raw.githubusercontent.com/llvm/llvm-project/main/cmake/Modules/LLVMVersion.cmake', ] - cmakelists = subprocess.run(curl_cmd, - capture_output=True, - check=True, - text=True).stdout + cmakelists = subprocess.run( + curl_cmd, capture_output=True, check=True, text=True + ).stdout if not (match := re.search(r'set\(LLVM_VERSION_MAJOR (\d+)', cmakelists)): raise RuntimeError('Could not find LLVM_VERSION_MAJOR?') @@ -64,9 +66,9 @@ def generate(config, tree): def check(trees_arg): try: - subprocess.run(['git', 'rev-parse', '--git-dir'], - check=True, - capture_output=True) + subprocess.run( + ['git', 'rev-parse', '--git-dir'], check=True, capture_output=True + ) except subprocess.CalledProcessError: # Print a nicer error message versus spewing the exception print('Script is not being run inside a git repository!') @@ -74,11 +76,13 @@ def check(trees_arg): if subprocess.run( ['git', '--no-optional-locks', 'status', '-uno', '--porcelain'], - capture_output=True, - check=True).stdout: + capture_output=True, + check=True, + ).stdout: print( f"\nRunning 'generate.py {trees_arg}' generated the following diff:\n", - flush=True) + flush=True, + ) subprocess.run(['git', '--no-pager', 'diff', 'HEAD'], check=True) print( "\nPlease run 'generate.py all' locally then commit and push the changes it creates!" diff --git a/generator/generate_tuxsuite.py b/generator/generate_tuxsuite.py index f88d834a..e1c89dd2 100755 --- a/generator/generate_tuxsuite.py +++ b/generator/generate_tuxsuite.py @@ -13,22 +13,31 @@ # pylint: disable-next=import-error import yaml -from utils import CI_ROOT, LLVM_TOT_VERSION, disable_subsys_werror_configs, get_config_from_generator, get_repo_ref, get_llvm_versions, patch_series_flag +from utils import ( + CI_ROOT, + LLVM_TOT_VERSION, + disable_subsys_werror_configs, + get_config_from_generator, + get_repo_ref, + get_llvm_versions, + patch_series_flag, +) # Aliases makes this YAML unreadable # https://ttl255.com/yaml-anchors-and-aliases-and-how-to-disable-them/ class NoAliasDumper(yaml.SafeDumper): - def ignore_aliases(self, _data): return True def parse_args(trees): parser = argparse.ArgumentParser(description="Generate TuxSuite YML.") - parser.add_argument("tree", - help="The git repo and ref to filter in.", - choices=[tree["name"] for tree in trees]) + parser.add_argument( + "tree", + help="The git repo and ref to filter in.", + choices=[tree["name"] for tree in trees], + ) return parser.parse_args() @@ -55,11 +64,8 @@ def emit_tuxsuite_yml(config, tree, llvm_version): # Input: '--patch-series ... ' # Output: '...' patches_folder = patches_flag.split(' ')[1] - print( - f"# $ git -C linux quiltimport --patches ../{patches_folder}") - print( - f"# $ scripts/build-local.py -C linux -f {tuxsuite_yml} -j defconfigs" - ) + print(f"# $ git -C linux quiltimport --patches ../{patches_folder}") + print(f"# $ scripts/build-local.py -C linux -f {tuxsuite_yml} -j defconfigs") tuxsuite_plan = { 'version': 1, @@ -71,15 +77,17 @@ def emit_tuxsuite_yml(config, tree, llvm_version): 'builds': [], } ] - } # yapf: disable + } # fmt: off max_version = int(LLVM_TOT_VERSION.read_text(encoding='utf-8')) defconfigs = [] distribution_configs = [] allconfigs = [] for build in config["builds"]: - if build["git_repo"] == repo and \ - build["git_ref"] == ref and \ - build["llvm_version"] == llvm_version: + if ( + build["git_repo"] == repo + and build["git_ref"] == ref + and build["llvm_version"] == llvm_version + ): arch = build.get("ARCH", "x86_64") if llvm_version == max_version: tuxsuite_toolchain = "clang-nightly" @@ -95,14 +103,12 @@ def emit_tuxsuite_yml(config, tree, llvm_version): "target_arch": arch, "toolchain": tuxsuite_toolchain, "kconfig": build["config"], - "targets": build["targets"] + "targets": build["targets"], } if "kernel_image" in build: - current_build.update( - {"kernel_image": build["kernel_image"]}) + current_build.update({"kernel_image": build["kernel_image"]}) if "make_variables" in build: - current_build.update( - {"make_variables": build["make_variables"]}) + current_build.update({"make_variables": build["make_variables"]}) cfg_str = str(build["config"]) if "defconfig" in cfg_str: @@ -114,20 +120,14 @@ def emit_tuxsuite_yml(config, tree, llvm_version): tuxsuite_plan["jobs"][0]["builds"] = defconfigs if distribution_configs: - tuxsuite_plan["jobs"] += [{ - "name": "distribution_configs", - "builds": distribution_configs - }] + tuxsuite_plan["jobs"] += [ + {"name": "distribution_configs", "builds": distribution_configs} + ] if allconfigs: - tuxsuite_plan["jobs"] += [{ - "name": "allconfigs", - "builds": allconfigs - }] + tuxsuite_plan["jobs"] += [{"name": "allconfigs", "builds": allconfigs}] print( - yaml.dump(tuxsuite_plan, - Dumper=NoAliasDumper, - width=1000, - sort_keys=False)) + yaml.dump(tuxsuite_plan, Dumper=NoAliasDumper, width=1000, sort_keys=False) + ) sys.stdout = orig_stdout diff --git a/generator/generate_workflow.py b/generator/generate_workflow.py index 9089e3c3..235785eb 100755 --- a/generator/generate_workflow.py +++ b/generator/generate_workflow.py @@ -15,15 +15,27 @@ # pylint: disable-next=import-error import yaml -from utils import CI_ROOT, LLVM_TOT_VERSION, disable_subsys_werror_configs, get_config_from_generator, get_llvm_versions, get_repo_ref, patch_series_flag, die +from utils import ( + CI_ROOT, + LLVM_TOT_VERSION, + disable_subsys_werror_configs, + get_config_from_generator, + get_llvm_versions, + get_repo_ref, + patch_series_flag, + die, +) def parse_args(trees): parser = argparse.ArgumentParser( - description="Generate GitHub Action Workflow YAML.") - parser.add_argument("tree", - help="The git repo and ref to filter in.", - choices=[tree["name"] for tree in trees]) + description="Generate GitHub Action Workflow YAML." + ) + parser.add_argument( + "tree", + help="The git repo and ref to filter in.", + choices=[tree["name"] for tree in trees], + ) return parser.parse_args() @@ -53,7 +65,7 @@ def initial_workflow(name, cron, tuxsuite_yml, workflow_yml): }, "permissions": "read-all", "jobs": {} - } # yapf: disable + } # fmt: off def print_config(build): @@ -109,7 +121,7 @@ def check_patches_job_setup(repo, ref, tree_name): }, ], } - } # yapf: disable + } # fmt: off def check_cache_job_setup(repo, ref, toolchain): @@ -161,13 +173,12 @@ def check_cache_job_setup(repo, ref, toolchain): }, ], } - } # yapf: disable + } # fmt: off def tuxsuite_setups(job_name, tuxsuite_yml, repo, ref): - patch_series = patch_series_flag( - tuxsuite_yml.split("/")[1].split("-clang-")[0]) - cond = {"if": "${{ needs.check_cache.outputs.output == 'failure' || github.event_name == 'workflow_dispatch' }}"} # yapf: disable + patch_series = patch_series_flag(tuxsuite_yml.split("/")[1].split("-clang-")[0]) + cond = {"if": "${{ needs.check_cache.outputs.output == 'failure' || github.event_name == 'workflow_dispatch' }}"} # fmt: off return { f"kick_tuxsuite_{job_name}": { "name": f"TuxSuite ({job_name})", @@ -236,7 +247,7 @@ def tuxsuite_setups(job_name, tuxsuite_yml, repo, ref): }, ] } - } # yapf: disable + } # fmt: off def get_steps(build, build_set): @@ -286,16 +297,14 @@ def get_steps(build, build_set): }, ], } - } # yapf: disable + } # fmt: off def get_cron_schedule(schedules, tree_name, llvm_version): for item in schedules: - if item["name"] == tree_name and \ - item["llvm_version"] == llvm_version: + if item["name"] == tree_name and item["llvm_version"] == llvm_version: return item["schedule"] - return die( - f"Could not find schedule for {tree_name} clang-{llvm_version}?") + return die(f"Could not find schedule for {tree_name} clang-{llvm_version}?") def print_builds(config, tree_name, llvm_version): @@ -308,39 +317,39 @@ def print_builds(config, tree_name, llvm_version): check_logs_distribution_configs = {} check_logs_allconfigs = {} for build in config["builds"]: - if build["git_repo"] == repo and \ - build["git_ref"] == ref and \ - build["llvm_version"] == llvm_version: + if ( + build["git_repo"] == repo + and build["git_ref"] == ref + and build["llvm_version"] == llvm_version + ): disable_subsys_werror_configs(build["config"]) cfg_str = str(build["config"]) if "defconfig" in cfg_str: check_logs_defconfigs.update(get_steps(build, "defconfigs")) elif "https://" in cfg_str: check_logs_distribution_configs.update( - get_steps(build, "distribution_configs")) + get_steps(build, "distribution_configs") + ) else: check_logs_allconfigs.update(get_steps(build, "allconfigs")) workflow_name = f"{tree_name} ({toolchain})" - cron_schedule = get_cron_schedule(config["tree_schedules"], tree_name, - llvm_version) - workflow = initial_workflow(workflow_name, cron_schedule, tuxsuite_yml, - github_yml) + cron_schedule = get_cron_schedule(config["tree_schedules"], tree_name, llvm_version) + workflow = initial_workflow(workflow_name, cron_schedule, tuxsuite_yml, github_yml) workflow['jobs'].update(check_patches_job_setup(repo, ref, tree_name)) workflow['jobs'].update(check_cache_job_setup(repo, ref, toolchain)) - workflow["jobs"].update( - tuxsuite_setups("defconfigs", tuxsuite_yml, repo, ref)) + workflow["jobs"].update(tuxsuite_setups("defconfigs", tuxsuite_yml, repo, ref)) workflow["jobs"].update(check_logs_defconfigs) if check_logs_distribution_configs: workflow["jobs"].update( - tuxsuite_setups("distribution_configs", tuxsuite_yml, repo, ref)) + tuxsuite_setups("distribution_configs", tuxsuite_yml, repo, ref) + ) workflow["jobs"].update(check_logs_distribution_configs) if check_logs_allconfigs: - workflow["jobs"].update( - tuxsuite_setups("allconfigs", tuxsuite_yml, repo, ref)) + workflow["jobs"].update(tuxsuite_setups("allconfigs", tuxsuite_yml, repo, ref)) workflow["jobs"].update(check_logs_allconfigs) with Path(CI_ROOT, github_yml).open("w", encoding='utf-8') as file: @@ -349,11 +358,7 @@ def print_builds(config, tree_name, llvm_version): print("# DO NOT MODIFY MANUALLY!") print("# This file has been autogenerated by invoking:") print(f"# $ ./generate_workflow.py {tree_name}") - print( - yaml.dump(workflow, - Dumper=yaml.Dumper, - width=1000, - sort_keys=False)) + print(yaml.dump(workflow, Dumper=yaml.Dumper, width=1000, sort_keys=False)) sys.stdout = orig_stdout diff --git a/ruff.toml b/ruff.toml index 31f51e65..27def937 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,6 +2,9 @@ target-version = 'py38' # Separate repo used as a submodule, ignore it extend-exclude = ['boot-utils'] +[format] +quote-style = 'preserve' + # https://docs.astral.sh/ruff/rules/ [lint] select = [ diff --git a/scripts/build-local.py b/scripts/build-local.py index c053dd5e..b9d4b847 100755 --- a/scripts/build-local.py +++ b/scripts/build-local.py @@ -42,42 +42,35 @@ def interrupt_handler(_signum, _frame): default_build_dir = Path(tuxmake_dir, 'build') default_output_dir = Path(tuxmake_dir, 'output') -parser = ArgumentParser( - description='Build TuxSuite YAML files locally using TuxMake') -parser.add_argument('-b', - '--build-dir', - default=default_build_dir, - help=f"Build folder (default: {default_build_dir})") -parser.add_argument('-c', - '--ccache', - action='store_true', - help='Use ccache if it is available') -parser.add_argument('-C', - '--directory', - help='Path to kernel source', - required=True) -parser.add_argument('-d', - '--debug', - action='store_true', - help='Show debugging messages') -parser.add_argument('-f', - '--files', - help='TuxSuite YAML files to build', - nargs='+', - required=True) -parser.add_argument('-j', - '--jobs', - help='Jobs to build (default: build all jobs)', - nargs='+') +parser = ArgumentParser(description='Build TuxSuite YAML files locally using TuxMake') +parser.add_argument( + '-b', + '--build-dir', + default=default_build_dir, + help=f"Build folder (default: {default_build_dir})", +) +parser.add_argument( + '-c', '--ccache', action='store_true', help='Use ccache if it is available' +) +parser.add_argument('-C', '--directory', help='Path to kernel source', required=True) +parser.add_argument( + '-d', '--debug', action='store_true', help='Show debugging messages' +) +parser.add_argument( + '-f', '--files', help='TuxSuite YAML files to build', nargs='+', required=True +) +parser.add_argument( + '-j', '--jobs', help='Jobs to build (default: build all jobs)', nargs='+' +) parser.add_argument( '-o', '--output-dir', default=default_output_dir, - help=f"Output folder for TuxMake files (default: {default_output_dir})") -parser.add_argument('-v', - '--verbose', - action='store_true', - help="Show tuxmake's output") + help=f"Output folder for TuxMake files (default: {default_output_dir})", +) +parser.add_argument( + '-v', '--verbose', action='store_true', help="Show tuxmake's output" +) args = parser.parse_args() if not (tree := Path(args.directory).resolve()).exists(): @@ -145,9 +138,11 @@ def interrupt_handler(_signum, _frame): ) cfg_str = '+'.join(kconfig) if isinstance(kconfig, list) else kconfig - print(f"I: Building {target_arch} {cfg_str} ({toolchain})... ", - end='\n' if args.verbose else '', - flush=True) + print( + f"I: Building {target_arch} {cfg_str} ({toolchain})... ", + end='\n' if args.verbose else '', + flush=True, + ) if build_dir.exists(): shutil.rmtree(build_dir) @@ -155,7 +150,7 @@ def interrupt_handler(_signum, _frame): # Replace the URL in the configuration string with a simple name, so # that it can be used in a path. - if (match := re.search(r'(https://[^\+]+)', cfg_str)): + if match := re.search(r'(https://[^\+]+)', cfg_str): url = match.groups()[0] if 'alpine' in url: distro = 'alpine' @@ -179,13 +174,15 @@ def interrupt_handler(_signum, _frame): build['kconfig'] = build['kconfig'][0] # Perform the build and report the result - result = tuxmake.build.build(build_dir=build_dir, - output_dir=specific_output_dir, - quiet=(not args.verbose), - runtime=runtime, - tree=tree, - wrapper=wrapper, - **build) + result = tuxmake.build.build( + build_dir=build_dir, + output_dir=specific_output_dir, + quiet=(not args.verbose), + runtime=runtime, + tree=tree, + wrapper=wrapper, + **build, + ) if all(info.passed for info in result.status.values()): print(f"{GREEN}PASS{NORMAL}") diff --git a/scripts/check-logs.py b/scripts/check-logs.py index 6cca7ed9..e0f756cc 100755 --- a/scripts/check-logs.py +++ b/scripts/check-logs.py @@ -17,7 +17,18 @@ import time import urllib.request -from utils import CI_ROOT, get_build, get_image_name, get_requested_llvm_version, print_red, get_cbl_name, show_builds, die, warn, info +from utils import ( + CI_ROOT, + get_build, + get_image_name, + get_requested_llvm_version, + print_red, + get_cbl_name, + show_builds, + die, + warn, + info, +) def _fetch(title, url, dest): @@ -164,9 +175,7 @@ def check_built_config(build): elif line.startswith("# CONFIG_"): name, state = line.split(" ", 2)[1:] if state != "is not set": - warn( - f"Could not parse '{name}' from .config line '{line}'!?" - ) + warn(f"Could not parse '{name}' from .config line '{line}'!?") state = 'n' elif not line.startswith("#"): warn(f"Could not parse .config line '{line}'!?") @@ -202,8 +211,10 @@ def print_clang_info(build): metadata_json = json.loads(metadata_file.read_text(encoding='utf-8')) info("Printing clang-nightly checkout date and hash") parse_cmd = [ - Path(CI_ROOT, "scripts/parse-debian-clang.py"), "--print-info", - "--version-string", metadata_json["compiler"]["version_full"] + Path(CI_ROOT, "scripts/parse-debian-clang.py"), + "--print-info", + "--version-string", + metadata_json["compiler"]["version_full"], ] subprocess.run(parse_cmd, check=True) @@ -238,16 +249,20 @@ def run_boot(build): ] # If we are running a sanitizer build, we should increase the number of # cores and timeout because booting is much slower - if "CONFIG_KASAN=y" in build["kconfig"] or \ - "CONFIG_KCSAN=y" in build["kconfig"] or \ - "CONFIG_UBSAN=y" in build["kconfig"]: + if ( + "CONFIG_KASAN=y" in build["kconfig"] + or "CONFIG_KCSAN=y" in build["kconfig"] + or "CONFIG_UBSAN=y" in build["kconfig"] + ): boot_cmd += ["-s", "4"] if "CONFIG_KASAN=y" in build["kconfig"]: boot_cmd += ["-t", "20m"] else: boot_cmd += ["-t", "10m"] - if "CONFIG_KASAN_KUNIT_TEST=y" in build["kconfig"] or \ - "CONFIG_KCSAN_KUNIT_TEST=y" in build["kconfig"]: + if ( + "CONFIG_KASAN_KUNIT_TEST=y" in build["kconfig"] + or "CONFIG_KCSAN_KUNIT_TEST=y" in build["kconfig"] + ): info("Disabling Oops problem matcher under Sanitizer KUnit build") print("::remove-matcher owner=linux-kernel-oopses::") diff --git a/scripts/check-patches-apply.py b/scripts/check-patches-apply.py index ca09ceed..569c8c86 100755 --- a/scripts/check-patches-apply.py +++ b/scripts/check-patches-apply.py @@ -9,21 +9,19 @@ from tempfile import TemporaryDirectory parser = ArgumentParser( - description='Check that patches apply to their tree before running CI') + description='Check that patches apply to their tree before running CI' +) parser.add_argument( '-p', '--patches-dir', help='Path to patches directory (can be relative or absolute)', required=True, - type=Path) -parser.add_argument('-r', - '--repo', - help='URL to git repository', - required=True) -parser.add_argument('-R', - '--ref', - help='Git reference to apply patches upon', - required=True) + type=Path, +) +parser.add_argument('-r', '--repo', help='URL to git repository', required=True) +parser.add_argument( + '-R', '--ref', help='Git reference to apply patches upon', required=True +) args = parser.parse_args() # If patches directory does not exist, there is nothing to check @@ -43,11 +41,10 @@ with TemporaryDirectory() as workdir: # Fetch the tarball from the repository. This is different for each type of # tree that we support. - if args.repo.startswith( - 'https://git.kernel.org/pub/scm/linux/kernel/git/'): + if args.repo.startswith('https://git.kernel.org/pub/scm/linux/kernel/git/'): # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git -> 'linux' if (base_repo := args.repo.rsplit('/', 1)[1]).endswith('.git'): - base_repo = base_repo[:-len('.git')] + base_repo = base_repo[: -len('.git')] # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git -> # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/snapshot/linux-master.tar.gz @@ -61,15 +58,16 @@ try: print(f"Downloading {tarball_url}...") - tarball = subprocess.run(['curl', '-LSs', tarball_url], - capture_output=True, - check=True).stdout + tarball = subprocess.run( + ['curl', '-LSs', tarball_url], capture_output=True, check=True + ).stdout print(f"Extracting {tarball_url}...") subprocess.run( ['tar', '-C', workdir, f"--strip-components={strip}", '-xzf-'], check=True, - input=tarball) + input=tarball, + ) except subprocess.CalledProcessError: print( 'Downloading or extracting tarball failed! As this may be flakiness on the server end, exiting 0 to have TuxSuite fail later...' @@ -87,7 +85,7 @@ 'GIT_AUTHOR_EMAIL': git_email, 'GIT_COMMITTER_NAME': git_name, 'GIT_COMMITTER_EMAIL': git_email, - } # yapf: disable + } # fmt: off print(f"Creating initial git repository in {workdir}...") subprocess.run(['git', 'init', '-q'], check=True, cwd=workdir) @@ -95,15 +93,18 @@ print(f"Adding files in {workdir}...") subprocess.run(['git', 'add', '.'], check=True, cwd=workdir) - print( - f"Creating initial commit '{args.repo} @ {args.ref}' in {workdir}...") - subprocess.run(['git', 'commit', '-m', f"{args.repo} @ {args.ref}", '-q'], - check=True, - cwd=workdir, - env=git_commit_env_vars) + print(f"Creating initial commit '{args.repo} @ {args.ref}' in {workdir}...") + subprocess.run( + ['git', 'commit', '-m', f"{args.repo} @ {args.ref}", '-q'], + check=True, + cwd=workdir, + env=git_commit_env_vars, + ) print(f"Applying patches in {workdir}...") - subprocess.run(['git', 'quiltimport', '--patches', patches_dir], - check=True, - cwd=workdir, - env=git_commit_env_vars) + subprocess.run( + ['git', 'quiltimport', '--patches', patches_dir], + check=True, + cwd=workdir, + env=git_commit_env_vars, + ) diff --git a/scripts/check-patches.py b/scripts/check-patches.py index c18985f5..05ab1a57 100755 --- a/scripts/check-patches.py +++ b/scripts/check-patches.py @@ -30,7 +30,9 @@ sys.exit(1) # A single line shell command to update a particular series file - update_series_command = f"\t$ ls -1 {folder}/*.patch | sed 's;{folder}/;;' > {folder}/series" + update_series_command = ( + f"\t$ ls -1 {folder}/*.patch | sed 's;{folder}/;;' > {folder}/series" + ) # Next, make sure series file is not missing if not (series := Path(folder, 'series')).exists(): @@ -55,8 +57,7 @@ # the series file (removed from series file, did not remove patch file) for patch in folder.glob('*.patch'): if patch.name not in series_text: - print( - f"{patch.name} not found in {series} but it is in {folder}?\n") + print(f"{patch.name} not found in {series} but it is in {folder}?\n") print('Update the series file:\n') print(update_series_command) sys.exit(1) @@ -77,9 +78,7 @@ sys.exit(1) if patch_folder.exists() and not wf_has_ps_opt: - print( - f"{patch_folder} exists but '--patch-series' not found in {workflow}?\n" - ) + print(f"{patch_folder} exists but '--patch-series' not found in {workflow}?\n") print('Regenerate the TuxSuite and workflow files:\n') print(f"\t$ ./generate.py {tree}\n") print('or remove the patches if they are no longer being used.') diff --git a/scripts/estimate-builds.py b/scripts/estimate-builds.py index da5294e4..7c5d2637 100755 --- a/scripts/estimate-builds.py +++ b/scripts/estimate-builds.py @@ -30,12 +30,13 @@ # Calculate the number of times that a workflow runs in a week based on its # schedule - num_runs = len( - list(croniter.croniter_range(now, week_from_now, tree['schedule']))) + num_runs = len(list(croniter.croniter_range(now, week_from_now, tree['schedule']))) for build in config['builds']: - if tree['git_repo'] == build['git_repo'] and \ - tree['git_ref'] == build['git_ref'] and \ - tree_llvm_ver == build['llvm_version']: + if ( + tree['git_repo'] == build['git_repo'] + and tree['git_ref'] == build['git_ref'] + and tree_llvm_ver == build['llvm_version'] + ): builds_per_tree[tree_name]['total'] += num_runs builds_per_tree[tree_name][tree_llvm_ver] += num_runs @@ -43,9 +44,9 @@ print(f"Total builds per week: {total_builds}") # Sort the list of builds by total number of builds descending -for tree, builds in sorted(builds_per_tree.items(), - key=lambda x: x[1]['total'], - reverse=True): +for tree, builds in sorted( + builds_per_tree.items(), key=lambda x: x[1]['total'], reverse=True +): print(f"\n - tree: {tree}") print(f" total: {builds['total']}") print(' breakdown:') diff --git a/scripts/generate-boot-utils-json.py b/scripts/generate-boot-utils-json.py index 75fe04ca..d6d68faa 100755 --- a/scripts/generate-boot-utils-json.py +++ b/scripts/generate-boot-utils-json.py @@ -12,14 +12,21 @@ repo = Path(__file__).resolve().parents[1] parser = ArgumentParser( - description='Download latest boot-utils release JSON from GitHub API') + description='Download latest boot-utils release JSON from GitHub API' +) parser.add_argument('github_token', help='Value of GITHUB_TOKEN') args = parser.parse_args() curl_cmd = [ - 'curl', '--header', 'Accept: application/vnd.github+json', '--header', - f"Authorization: Bearer {args.github_token}", '--output', - Path(repo, 'boot-utils.json'), '--silent', '--show-error', - 'https://api.github.com/repos/ClangBuiltLinux/boot-utils/releases/latest' + 'curl', + '--header', + 'Accept: application/vnd.github+json', + '--header', + f"Authorization: Bearer {args.github_token}", + '--output', + Path(repo, 'boot-utils.json'), + '--silent', + '--show-error', + 'https://api.github.com/repos/ClangBuiltLinux/boot-utils/releases/latest', ] subprocess.run(curl_cmd, check=True) diff --git a/scripts/markdown-badges.py b/scripts/markdown-badges.py index 8eed6add..236b929c 100755 --- a/scripts/markdown-badges.py +++ b/scripts/markdown-badges.py @@ -23,7 +23,6 @@ def order_to_rank(iterable, item): class ClangVersion(_BaseVersion): - def __init__(self, version): if 'clang-' not in version: raise ValueError(f"Invalid clang version ('{version}') provided?") @@ -35,7 +34,6 @@ def __init__(self, version): class KernelVersion(_BaseVersion): - def __init__(self, version): major = 0 minor = 0 @@ -52,12 +50,12 @@ def __init__(self, version): major = order_to_rank(upstream_trees, version) # LTS releases - if (match := re.search(r'^([\d|\.]+)$', version)): + if match := re.search(r'^([\d|\.]+)$', version): category = 'lts' major, minor = map(int, match.groups()[0].split('.')) # Named maintainer trees - maintainer_trees = ('tip', ) + maintainer_trees = ('tip',) if version in maintainer_trees: category = 'maintainers' major = order_to_rank(maintainer_trees, version) @@ -146,10 +144,16 @@ def svg(workflow): # - Make every column the same width # - Force whitespace to non-breaking (to keep the cell from eliding spaces) # - Replace "-" with non-breaking-dash (to keep the name from wrapping) -print("| | " + " | ".join([ - " " * (max_width - len(col)) + col.replace('-', '‑') - for col in columns -]) + " |") +print( + "| | " + + " | ".join( + [ + " " * (max_width - len(col)) + col.replace('-', '‑') + for col in columns + ] + ) + + " |" +) # Align tree name to the right for readability, and center the badges. print("| ---: |" + " :---: |" * len(columns)) @@ -165,13 +169,14 @@ def svg(workflow): for tree in rows: # Keep names from wrapping. row = tree.replace('-', '‑') - print(f"| {row} | " + " | ".join( - [svg(trees[tree].get(compiler, None)) for compiler in columns]) + " |") + print( + f"| {row} | " + + " | ".join([svg(trees[tree].get(compiler, None)) for compiler in columns]) + + " |" + ) # Output a button for the "Check clang version" workflow, which ensures that # tip of tree LLVM is being updating. This does not need to be a part of the # table above. cv_workflow_url = 'https://github.com/clangbuiltlinux/continuous-integration2/actions/workflows/clang-version.yml' -print( - f"\n[![Check clang version]({cv_workflow_url}/badge.svg)]({cv_workflow_url})" -) +print(f"\n[![Check clang version]({cv_workflow_url}/badge.svg)]({cv_workflow_url})") diff --git a/scripts/parse-debian-clang.py b/scripts/parse-debian-clang.py index eb448384..ec9fee76 100755 --- a/scripts/parse-debian-clang.py +++ b/scripts/parse-debian-clang.py @@ -7,25 +7,29 @@ import subprocess parser = ArgumentParser(description="Parse Debian's clang version") -parser.add_argument('-c', - '--check', - action='store_true', - help='Fail if clang has not been updated in 5 days') -parser.add_argument('-p', - '--print-info', - action='store_true', - help='Print information about clang version') +parser.add_argument( + '-c', + '--check', + action='store_true', + help='Fail if clang has not been updated in 5 days', +) +parser.add_argument( + '-p', + '--print-info', + action='store_true', + help='Print information about clang version', +) parser.add_argument( '-v', '--version-string', - help="Use value as clang version instead of calling 'clang --version'") + help="Use value as clang version instead of calling 'clang --version'", +) args = parser.parse_args() if not (version_string := args.version_string): - clang_version = subprocess.run(['clang', '--version'], - capture_output=True, - check=True, - text=True).stdout + clang_version = subprocess.run( + ['clang', '--version'], capture_output=True, check=True, text=True + ).stdout version_string = clang_version.splitlines()[0] # $ clang-14 --version | head -1 diff --git a/scripts/visualize-builds.py b/scripts/visualize-builds.py index 9026e675..d70bb574 100755 --- a/scripts/visualize-builds.py +++ b/scripts/visualize-builds.py @@ -119,27 +119,29 @@ def output_pretty_table(days, output_file=None): for day, hours in days.items(): col_widths[0] = max(col_widths[0], len(CRON_TO_DAY[day])) for hour in range(HOURS): - col_widths[hour + 1] = max(col_widths[hour + 1], - len(str(len(hours[hour])))) + col_widths[hour + 1] = max(col_widths[hour + 1], len(str(len(hours[hour])))) top_border = "┌" + "┬".join("─" * width for width in col_widths) + "┐" print(top_border, file=output) - header_row = ("│" + - "│".join(f"{cell:^{width}}" - for cell, width in zip(header_cells, col_widths)) + - "│") + header_row = ( + "│" + + "│".join(f"{cell:^{width}}" for cell, width in zip(header_cells, col_widths)) + + "│" + ) print(header_row, file=output) header_sep = "├" + "┼".join("─" * width for width in col_widths) + "┤" print(header_sep, file=output) for day, hours in days.items(): - row_cells = [CRON_TO_DAY[day] - ] + [str(len(hours[hour])) for hour in range(HOURS)] - data_row = ("│" + - "│".join(f"{cell:^{width}}" - for cell, width in zip(row_cells, col_widths)) + - "│") + row_cells = [CRON_TO_DAY[day]] + [ + str(len(hours[hour])) for hour in range(HOURS) + ] + data_row = ( + "│" + + "│".join(f"{cell:^{width}}" for cell, width in zip(row_cells, col_widths)) + + "│" + ) print(data_row, file=output) # Bottom border @@ -149,17 +151,17 @@ def output_pretty_table(days, output_file=None): def parse_args(): parser = argparse.ArgumentParser( - description="Visualize build distribution across days and hours") + description="Visualize build distribution across days and hours" + ) parser.add_argument( "--format", choices=["table", "csv", "pretty"], default="pretty", help="Output format (default: pretty)", ) - parser.add_argument("--output", - "-o", - type=str, - help="Output file (default: stdout)") + parser.add_argument( + "--output", "-o", type=str, help="Output file (default: stdout)" + ) return parser.parse_args() @@ -173,7 +175,8 @@ def main(): if args.output: # pylint: disable-next=consider-using-with output_file = open( # noqa: SIM115 - args.output, "w", encoding="utf-8", newline="") + args.output, "w", encoding="utf-8", newline="" + ) try: if args.format == "csv": diff --git a/utils.py b/utils.py index c1883467..78d9bfdd 100644 --- a/utils.py +++ b/utils.py @@ -36,8 +36,7 @@ def disable_subsys_werror_configs(configs): def get_config_from_generator(): - if not (all_generator_files := sorted( - Path(GENERATOR_ROOT, 'yml').glob('*.yml'))): + if not (all_generator_files := sorted(Path(GENERATOR_ROOT, 'yml').glob('*.yml'))): return die('No generator files could not be found?') generator_pieces = [] @@ -55,11 +54,9 @@ def get_image_name(): arch = os.environ["ARCH"] if arch == "powerpc": subarch = get_cbl_name() - return { - "ppc32": "uImage", - "ppc64": "vmlinux", - "ppc64le": "zImage.epapr" - }[subarch] + return {"ppc32": "uImage", "ppc64": "vmlinux", "ppc64le": "zImage.epapr"}[ + subarch + ] return { "arm": "zImage", "arm64": "Image.gz", @@ -86,7 +83,7 @@ def get_cbl_name(): "aarch64": "arm64", "armv7": "arm32_v7", "riscv64": "riscv", - "x86_64": "x86_64" + "x86_64": "x86_64", } # The URL is https://.../stable..config alpine_arch = base_config.split(".")[-2] @@ -98,7 +95,7 @@ def get_cbl_name(): "i686": "x86", "ppc64le": "ppc64le", "s390x": "s390", - "x86_64": "x86_64" + "x86_64": "x86_64", } # The URL is https://.../kernel--fedora.config fedora_arch = base_config.split("/")[-1].split("-")[1] @@ -111,7 +108,7 @@ def get_cbl_name(): "ppc64le": "ppc64le", "riscv64": "riscv", "s390x": "s390", - "x86_64": "x86_64" + "x86_64": "x86_64", } # The URL is https://...//default suse_arch = base_config.split("/")[-2] @@ -184,9 +181,11 @@ def get_build(): configs = os.environ["CONFIG"].split("+") llvm_version = get_requested_llvm_version() for build in _read_builds(): - if build["target_arch"] == arch and \ - build["toolchain"] == llvm_version and \ - build["kconfig"] == configs: + if ( + build["target_arch"] == arch + and build["toolchain"] == llvm_version + and build["kconfig"] == configs + ): return build print_red("ERROR: Unable to find build") show_builds() @@ -211,8 +210,7 @@ def get_llvm_versions(config, tree_name): def get_patches_hash(tree_name): patches_folder = Path(CI_ROOT, 'patches', tree_name) - patches = sorted( - patches_folder.iterdir()) if patches_folder.exists() else [] + patches = sorted(patches_folder.iterdir()) if patches_folder.exists() else [] text = ''.join(item.read_text(encoding='utf-8') for item in patches) @@ -242,7 +240,7 @@ def update_repository_variable( patches_hash: Optional[str] = None, build_status: Optional[str] = None, other: Optional[Dict[str, str]] = None, - allow_fail_to_pass=False # should a cache entry be allowed to go from 'fail' to 'pass' + allow_fail_to_pass=False, # should a cache entry be allowed to go from 'fail' to 'pass' ): """ Update cache entries. @@ -270,8 +268,11 @@ def update_repository_variable( if patches_hash: cached_value["patches_hash"] = patches_hash if build_status: - if not allow_fail_to_pass and cached_value[ - 'build_status'] == 'fail' and build_status == 'pass': + if ( + not allow_fail_to_pass + and cached_value['build_status'] == 'fail' + and build_status == 'pass' + ): ... else: cached_value["build_status"] = build_status @@ -281,14 +282,10 @@ def update_repository_variable( cached_value = json.dumps(cached_value) - new_value = json.dumps({ - "name": key, - "value": cached_value - }).encode("utf-8") - update_request = urllib.request.Request(url, - data=new_value, - method="PATCH", - headers=http_headers) + new_value = json.dumps({"name": key, "value": cached_value}).encode("utf-8") + update_request = urllib.request.Request( + url, data=new_value, method="PATCH", headers=http_headers + ) urllib.request.urlopen(update_request) # pylint: disable=consider-using-with print(f"""\