diff --git a/caching/check.py b/caching/check.py index 3b106740..7ac893a9 100755 --- a/caching/check.py +++ b/caching/check.py @@ -36,6 +36,7 @@ we want to only cache on a "pass" or "fail" status as these mean that Tuxsuite actually completed its work and didnt timeout. """ + import argparse import json import os @@ -48,7 +49,11 @@ # pylint: disable-next=import-error import requests -from utils import get_patches_hash, get_workflow_name_to_var_name, update_repository_variable +from utils import ( + get_patches_hash, + get_workflow_name_to_var_name, + update_repository_variable, +) OWNER = "ClangBuiltLinux" REPO = "continuous-integration2" @@ -63,8 +68,7 @@ TIMEOUT = 64 -class MalformedCacheError(Exception): - ... +class MalformedCacheError(Exception): ... def parse_args(): @@ -107,7 +111,8 @@ def ___purge___cache___(): list_response = requests.get(list_url, headers=headers, timeout=TIMEOUT) print(list_response.content) all_variables_keys = [ - x["name"] for x in json.loads(list_response.content)["variables"] + x["name"] + for x in json.loads(list_response.content)["variables"] if x["name"].startswith("_") ] @@ -115,9 +120,7 @@ def ___purge___cache___(): delete_url = ( f"https://api.github.com/repos/{OWNER}/{REPO}/actions/variables/{key}" ) - delete_response = requests.delete(delete_url, - headers=headers, - timeout=TIMEOUT) + delete_response = requests.delete(delete_url, headers=headers, timeout=TIMEOUT) if delete_response.status_code != 204: print(f"ERROR: Couldn't delete cache entry with key {key}") sys.exit(1) @@ -154,16 +157,19 @@ def get_repository_variable_or_none(name: str) -> Optional[dict]: return json.loads(as_dict["value"]) -def create_repository_variable(name: str, linux_sha: str, clang_version: str, - patches_hash: str) -> None: +def create_repository_variable( + name: str, linux_sha: str, clang_version: str, patches_hash: str +) -> None: _url = f"https://api.github.com/repos/{OWNER}/{REPO}/actions/variables" - _value = json.dumps({ - "linux_sha": linux_sha, - "clang_version": clang_version, - "patches_hash": patches_hash, - "build_status": "presuite", - }) + _value = json.dumps( + { + "linux_sha": linux_sha, + "clang_version": clang_version, + "patches_hash": patches_hash, + "build_status": "presuite", + } + ) data = {"name": name, "value": _value} resp = requests.post(_url, headers=headers, json=data, timeout=TIMEOUT) @@ -199,8 +205,10 @@ def create_repository_variable(name: str, linux_sha: str, clang_version: str, # pull down repo variable result = get_repository_variable_or_none(VAR_NAME) if result is None: - print(f"CACHE MISS: Did not find repo variable {VAR_NAME} " - f"from workflow_name: {args.workflow_name}. Creating it now.") + print( + f"CACHE MISS: Did not find repo variable {VAR_NAME} " + f"from workflow_name: {args.workflow_name}. Creating it now." + ) create_repository_variable( VAR_NAME, linux_sha=curr_sha, @@ -219,14 +227,19 @@ def create_repository_variable(name: str, linux_sha: str, clang_version: str, raise MalformedCacheError( f"The cache with key {VAR_NAME} based on workflow '{args.workflow_name}' " f"is one or more fields. It's missing: {missing_fields}\n" - f"The current cache looks as follows:\n{result}.") + f"The current cache looks as follows:\n{result}." + ) cached_sha = result["linux_sha"] cached_clang_version = result["clang_version"] cached_build_status = result["build_status"] cached_patches_hash = result.get("patches_hash", curr_patches_hash) - if cached_sha != curr_sha or cached_clang_version != curr_clang_version or cached_patches_hash != curr_patches_hash: + if ( + cached_sha != curr_sha + or cached_clang_version != curr_clang_version + or cached_patches_hash != curr_patches_hash + ): print(f"""\ CACHE MISS: current linux_sha is {curr_sha}, clang_version is {curr_clang_version}, and current patches_hash is {curr_patches_hash} while {args.workflow_name} has @@ -271,5 +284,4 @@ def create_repository_variable(name: str, linux_sha: str, clang_version: str, with open(env_file, "a", encoding="utf-8") as fd: fd.write(f"CACHE_PASS={cached_build_status.strip()}") - sys.exit( - 0) # signifies to the workflow that no jobs should run ('success') + sys.exit(0) # signifies to the workflow that no jobs should run ('success') diff --git a/caching/update.py b/caching/update.py index 46975c27..33bfb1c0 100755 --- a/caching/update.py +++ b/caching/update.py @@ -29,7 +29,11 @@ import urllib.request from pathlib import Path -from utils import get_patches_hash, get_workflow_name_to_var_name, update_repository_variable +from utils import ( + get_patches_hash, + get_workflow_name_to_var_name, + update_repository_variable, +) if "GITHUB_WORKFLOW" not in os.environ: print("Couldn't find GITHUB_WORKFLOW in env. Not in a GitHub Workflow?") @@ -38,15 +42,12 @@ MOCK = "MOCK" in os.environ -def update_cache(status: str, git_sha: str, clang_version: str, - patches_hash: str): +def update_cache(status: str, git_sha: str, clang_version: str, patches_hash: str): print(f"Trying to update cache with status: {status}") - cache_entry_key = get_workflow_name_to_var_name( - os.environ["GITHUB_WORKFLOW"]) + cache_entry_key = get_workflow_name_to_var_name(os.environ["GITHUB_WORKFLOW"]) if "REPO_SCOPED_PAT" not in os.environ: - print( - "Couldn't find REPO_SCOPED_PAT in env. Not in a GitHub Workflow?") + print("Couldn't find REPO_SCOPED_PAT in env. Not in a GitHub Workflow?") sys.exit(1) headers = {"Authorization": f"Bearer {os.environ['REPO_SCOPED_PAT']}"} @@ -82,8 +83,7 @@ def main(): for entry, build in builds.items(): try: git_sha = build["git_sha"] - clang_version = build["tuxmake_metadata"]["compiler"][ - "version_full"] + clang_version = build["tuxmake_metadata"]["compiler"]["version_full"] break except KeyError: builds_that_are_missing_metadata.append(entry) @@ -100,32 +100,32 @@ def main(): build_log_raw = response.read().decode() failed_pattern = ( - r"(?<=Apply patch set FAILED\s)[0-9A-Za-z._:/\-\s]*?(?=\serror: )") + r"(?<=Apply patch set FAILED\s)[0-9A-Za-z._:/\-\s]*?(?=\serror: )" + ) failed_matches = re.findall(failed_pattern, build_log_raw) if len(failed_matches) == 0: print( f"No patches failed to apply yet the build status stated there were: {build['status_message']}" ) - sys.exit( - 0) # Not sure how we got here but continue the action anyways + sys.exit(0) # Not sure how we got here but continue the action anyways patches_that_failed_to_apply = failed_matches[0].split('\n') - print( - f"Error: Some patches failed to apply.\n{patches_that_failed_to_apply}\n" - ) + print(f"Error: Some patches failed to apply.\n{patches_that_failed_to_apply}\n") sys.exit(1) if len(builds_that_are_missing_metadata) == len(builds): raise RuntimeError( f"Could not find a suitable git sha or compiler version in any build\n" - f"Here's the build.json:\n{raw}") + f"Here's the build.json:\n{raw}" + ) if len(builds_that_are_missing_metadata) > 0: print( "Warning: Some of the builds in builds.json are malformed and missing " "some metadata.\n" f"Here's a list: {builds_that_are_missing_metadata}\n" - f"Here's the build.json in question:\n{raw}") + f"Here's the build.json in question:\n{raw}" + ) assert git_sha and clang_version diff --git a/generator/generate.py b/generator/generate.py index e28fb965..b306f88c 100755 --- a/generator/generate.py +++ b/generator/generate.py @@ -17,20 +17,22 @@ def parse_args(trees): - parser = ArgumentParser( - description='Generate yml files and perform extra checks') + parser = ArgumentParser(description='Generate yml files and perform extra checks') - parser.add_argument('-c', - '--check', - action='store_true', - help='Fail if generating yml files results in a diff') + parser.add_argument( + '-c', + '--check', + action='store_true', + help='Fail if generating yml files results in a diff', + ) parser.add_argument( 'trees', choices=[*trees, 'all'], default='all', help='The trees to generate yml files for (default: all)', metavar='TREES', - nargs='*') + nargs='*', + ) return parser.parse_args() @@ -38,13 +40,13 @@ def parse_args(trees): def update_llvm_tot_version(): # Avoids pulling in an extra Python package dependency curl_cmd = [ - 'curl', '-fLSs', - 'https://raw.githubusercontent.com/llvm/llvm-project/main/cmake/Modules/LLVMVersion.cmake' + 'curl', + '-fLSs', + 'https://raw.githubusercontent.com/llvm/llvm-project/main/cmake/Modules/LLVMVersion.cmake', ] - cmakelists = subprocess.run(curl_cmd, - capture_output=True, - check=True, - text=True).stdout + cmakelists = subprocess.run( + curl_cmd, capture_output=True, check=True, text=True + ).stdout if not (match := re.search(r'set\(LLVM_VERSION_MAJOR (\d+)', cmakelists)): raise RuntimeError('Could not find LLVM_VERSION_MAJOR?') @@ -64,9 +66,9 @@ def generate(config, tree): def check(trees_arg): try: - subprocess.run(['git', 'rev-parse', '--git-dir'], - check=True, - capture_output=True) + subprocess.run( + ['git', 'rev-parse', '--git-dir'], check=True, capture_output=True + ) except subprocess.CalledProcessError: # Print a nicer error message versus spewing the exception print('Script is not being run inside a git repository!') @@ -74,11 +76,13 @@ def check(trees_arg): if subprocess.run( ['git', '--no-optional-locks', 'status', '-uno', '--porcelain'], - capture_output=True, - check=True).stdout: + capture_output=True, + check=True, + ).stdout: print( f"\nRunning 'generate.py {trees_arg}' generated the following diff:\n", - flush=True) + flush=True, + ) subprocess.run(['git', '--no-pager', 'diff', 'HEAD'], check=True) print( "\nPlease run 'generate.py all' locally then commit and push the changes it creates!" diff --git a/generator/generate_tuxsuite.py b/generator/generate_tuxsuite.py index f88d834a..e1c89dd2 100755 --- a/generator/generate_tuxsuite.py +++ b/generator/generate_tuxsuite.py @@ -13,22 +13,31 @@ # pylint: disable-next=import-error import yaml -from utils import CI_ROOT, LLVM_TOT_VERSION, disable_subsys_werror_configs, get_config_from_generator, get_repo_ref, get_llvm_versions, patch_series_flag +from utils import ( + CI_ROOT, + LLVM_TOT_VERSION, + disable_subsys_werror_configs, + get_config_from_generator, + get_repo_ref, + get_llvm_versions, + patch_series_flag, +) # Aliases makes this YAML unreadable # https://ttl255.com/yaml-anchors-and-aliases-and-how-to-disable-them/ class NoAliasDumper(yaml.SafeDumper): - def ignore_aliases(self, _data): return True def parse_args(trees): parser = argparse.ArgumentParser(description="Generate TuxSuite YML.") - parser.add_argument("tree", - help="The git repo and ref to filter in.", - choices=[tree["name"] for tree in trees]) + parser.add_argument( + "tree", + help="The git repo and ref to filter in.", + choices=[tree["name"] for tree in trees], + ) return parser.parse_args() @@ -55,11 +64,8 @@ def emit_tuxsuite_yml(config, tree, llvm_version): # Input: '--patch-series ... ' # Output: '...' patches_folder = patches_flag.split(' ')[1] - print( - f"# $ git -C linux quiltimport --patches ../{patches_folder}") - print( - f"# $ scripts/build-local.py -C linux -f {tuxsuite_yml} -j defconfigs" - ) + print(f"# $ git -C linux quiltimport --patches ../{patches_folder}") + print(f"# $ scripts/build-local.py -C linux -f {tuxsuite_yml} -j defconfigs") tuxsuite_plan = { 'version': 1, @@ -71,15 +77,17 @@ def emit_tuxsuite_yml(config, tree, llvm_version): 'builds': [], } ] - } # yapf: disable + } # fmt: off max_version = int(LLVM_TOT_VERSION.read_text(encoding='utf-8')) defconfigs = [] distribution_configs = [] allconfigs = [] for build in config["builds"]: - if build["git_repo"] == repo and \ - build["git_ref"] == ref and \ - build["llvm_version"] == llvm_version: + if ( + build["git_repo"] == repo + and build["git_ref"] == ref + and build["llvm_version"] == llvm_version + ): arch = build.get("ARCH", "x86_64") if llvm_version == max_version: tuxsuite_toolchain = "clang-nightly" @@ -95,14 +103,12 @@ def emit_tuxsuite_yml(config, tree, llvm_version): "target_arch": arch, "toolchain": tuxsuite_toolchain, "kconfig": build["config"], - "targets": build["targets"] + "targets": build["targets"], } if "kernel_image" in build: - current_build.update( - {"kernel_image": build["kernel_image"]}) + current_build.update({"kernel_image": build["kernel_image"]}) if "make_variables" in build: - current_build.update( - {"make_variables": build["make_variables"]}) + current_build.update({"make_variables": build["make_variables"]}) cfg_str = str(build["config"]) if "defconfig" in cfg_str: @@ -114,20 +120,14 @@ def emit_tuxsuite_yml(config, tree, llvm_version): tuxsuite_plan["jobs"][0]["builds"] = defconfigs if distribution_configs: - tuxsuite_plan["jobs"] += [{ - "name": "distribution_configs", - "builds": distribution_configs - }] + tuxsuite_plan["jobs"] += [ + {"name": "distribution_configs", "builds": distribution_configs} + ] if allconfigs: - tuxsuite_plan["jobs"] += [{ - "name": "allconfigs", - "builds": allconfigs - }] + tuxsuite_plan["jobs"] += [{"name": "allconfigs", "builds": allconfigs}] print( - yaml.dump(tuxsuite_plan, - Dumper=NoAliasDumper, - width=1000, - sort_keys=False)) + yaml.dump(tuxsuite_plan, Dumper=NoAliasDumper, width=1000, sort_keys=False) + ) sys.stdout = orig_stdout diff --git a/generator/generate_workflow.py b/generator/generate_workflow.py index 9089e3c3..235785eb 100755 --- a/generator/generate_workflow.py +++ b/generator/generate_workflow.py @@ -15,15 +15,27 @@ # pylint: disable-next=import-error import yaml -from utils import CI_ROOT, LLVM_TOT_VERSION, disable_subsys_werror_configs, get_config_from_generator, get_llvm_versions, get_repo_ref, patch_series_flag, die +from utils import ( + CI_ROOT, + LLVM_TOT_VERSION, + disable_subsys_werror_configs, + get_config_from_generator, + get_llvm_versions, + get_repo_ref, + patch_series_flag, + die, +) def parse_args(trees): parser = argparse.ArgumentParser( - description="Generate GitHub Action Workflow YAML.") - parser.add_argument("tree", - help="The git repo and ref to filter in.", - choices=[tree["name"] for tree in trees]) + description="Generate GitHub Action Workflow YAML." + ) + parser.add_argument( + "tree", + help="The git repo and ref to filter in.", + choices=[tree["name"] for tree in trees], + ) return parser.parse_args() @@ -53,7 +65,7 @@ def initial_workflow(name, cron, tuxsuite_yml, workflow_yml): }, "permissions": "read-all", "jobs": {} - } # yapf: disable + } # fmt: off def print_config(build): @@ -109,7 +121,7 @@ def check_patches_job_setup(repo, ref, tree_name): }, ], } - } # yapf: disable + } # fmt: off def check_cache_job_setup(repo, ref, toolchain): @@ -161,13 +173,12 @@ def check_cache_job_setup(repo, ref, toolchain): }, ], } - } # yapf: disable + } # fmt: off def tuxsuite_setups(job_name, tuxsuite_yml, repo, ref): - patch_series = patch_series_flag( - tuxsuite_yml.split("/")[1].split("-clang-")[0]) - cond = {"if": "${{ needs.check_cache.outputs.output == 'failure' || github.event_name == 'workflow_dispatch' }}"} # yapf: disable + patch_series = patch_series_flag(tuxsuite_yml.split("/")[1].split("-clang-")[0]) + cond = {"if": "${{ needs.check_cache.outputs.output == 'failure' || github.event_name == 'workflow_dispatch' }}"} # fmt: off return { f"kick_tuxsuite_{job_name}": { "name": f"TuxSuite ({job_name})", @@ -236,7 +247,7 @@ def tuxsuite_setups(job_name, tuxsuite_yml, repo, ref): }, ] } - } # yapf: disable + } # fmt: off def get_steps(build, build_set): @@ -286,16 +297,14 @@ def get_steps(build, build_set): }, ], } - } # yapf: disable + } # fmt: off def get_cron_schedule(schedules, tree_name, llvm_version): for item in schedules: - if item["name"] == tree_name and \ - item["llvm_version"] == llvm_version: + if item["name"] == tree_name and item["llvm_version"] == llvm_version: return item["schedule"] - return die( - f"Could not find schedule for {tree_name} clang-{llvm_version}?") + return die(f"Could not find schedule for {tree_name} clang-{llvm_version}?") def print_builds(config, tree_name, llvm_version): @@ -308,39 +317,39 @@ def print_builds(config, tree_name, llvm_version): check_logs_distribution_configs = {} check_logs_allconfigs = {} for build in config["builds"]: - if build["git_repo"] == repo and \ - build["git_ref"] == ref and \ - build["llvm_version"] == llvm_version: + if ( + build["git_repo"] == repo + and build["git_ref"] == ref + and build["llvm_version"] == llvm_version + ): disable_subsys_werror_configs(build["config"]) cfg_str = str(build["config"]) if "defconfig" in cfg_str: check_logs_defconfigs.update(get_steps(build, "defconfigs")) elif "https://" in cfg_str: check_logs_distribution_configs.update( - get_steps(build, "distribution_configs")) + get_steps(build, "distribution_configs") + ) else: check_logs_allconfigs.update(get_steps(build, "allconfigs")) workflow_name = f"{tree_name} ({toolchain})" - cron_schedule = get_cron_schedule(config["tree_schedules"], tree_name, - llvm_version) - workflow = initial_workflow(workflow_name, cron_schedule, tuxsuite_yml, - github_yml) + cron_schedule = get_cron_schedule(config["tree_schedules"], tree_name, llvm_version) + workflow = initial_workflow(workflow_name, cron_schedule, tuxsuite_yml, github_yml) workflow['jobs'].update(check_patches_job_setup(repo, ref, tree_name)) workflow['jobs'].update(check_cache_job_setup(repo, ref, toolchain)) - workflow["jobs"].update( - tuxsuite_setups("defconfigs", tuxsuite_yml, repo, ref)) + workflow["jobs"].update(tuxsuite_setups("defconfigs", tuxsuite_yml, repo, ref)) workflow["jobs"].update(check_logs_defconfigs) if check_logs_distribution_configs: workflow["jobs"].update( - tuxsuite_setups("distribution_configs", tuxsuite_yml, repo, ref)) + tuxsuite_setups("distribution_configs", tuxsuite_yml, repo, ref) + ) workflow["jobs"].update(check_logs_distribution_configs) if check_logs_allconfigs: - workflow["jobs"].update( - tuxsuite_setups("allconfigs", tuxsuite_yml, repo, ref)) + workflow["jobs"].update(tuxsuite_setups("allconfigs", tuxsuite_yml, repo, ref)) workflow["jobs"].update(check_logs_allconfigs) with Path(CI_ROOT, github_yml).open("w", encoding='utf-8') as file: @@ -349,11 +358,7 @@ def print_builds(config, tree_name, llvm_version): print("# DO NOT MODIFY MANUALLY!") print("# This file has been autogenerated by invoking:") print(f"# $ ./generate_workflow.py {tree_name}") - print( - yaml.dump(workflow, - Dumper=yaml.Dumper, - width=1000, - sort_keys=False)) + print(yaml.dump(workflow, Dumper=yaml.Dumper, width=1000, sort_keys=False)) sys.stdout = orig_stdout diff --git a/ruff.toml b/ruff.toml index 31f51e65..27def937 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,6 +2,9 @@ target-version = 'py38' # Separate repo used as a submodule, ignore it extend-exclude = ['boot-utils'] +[format] +quote-style = 'preserve' + # https://docs.astral.sh/ruff/rules/ [lint] select = [ diff --git a/scripts/build-local.py b/scripts/build-local.py index c053dd5e..b9d4b847 100755 --- a/scripts/build-local.py +++ b/scripts/build-local.py @@ -42,42 +42,35 @@ def interrupt_handler(_signum, _frame): default_build_dir = Path(tuxmake_dir, 'build') default_output_dir = Path(tuxmake_dir, 'output') -parser = ArgumentParser( - description='Build TuxSuite YAML files locally using TuxMake') -parser.add_argument('-b', - '--build-dir', - default=default_build_dir, - help=f"Build folder (default: {default_build_dir})") -parser.add_argument('-c', - '--ccache', - action='store_true', - help='Use ccache if it is available') -parser.add_argument('-C', - '--directory', - help='Path to kernel source', - required=True) -parser.add_argument('-d', - '--debug', - action='store_true', - help='Show debugging messages') -parser.add_argument('-f', - '--files', - help='TuxSuite YAML files to build', - nargs='+', - required=True) -parser.add_argument('-j', - '--jobs', - help='Jobs to build (default: build all jobs)', - nargs='+') +parser = ArgumentParser(description='Build TuxSuite YAML files locally using TuxMake') +parser.add_argument( + '-b', + '--build-dir', + default=default_build_dir, + help=f"Build folder (default: {default_build_dir})", +) +parser.add_argument( + '-c', '--ccache', action='store_true', help='Use ccache if it is available' +) +parser.add_argument('-C', '--directory', help='Path to kernel source', required=True) +parser.add_argument( + '-d', '--debug', action='store_true', help='Show debugging messages' +) +parser.add_argument( + '-f', '--files', help='TuxSuite YAML files to build', nargs='+', required=True +) +parser.add_argument( + '-j', '--jobs', help='Jobs to build (default: build all jobs)', nargs='+' +) parser.add_argument( '-o', '--output-dir', default=default_output_dir, - help=f"Output folder for TuxMake files (default: {default_output_dir})") -parser.add_argument('-v', - '--verbose', - action='store_true', - help="Show tuxmake's output") + help=f"Output folder for TuxMake files (default: {default_output_dir})", +) +parser.add_argument( + '-v', '--verbose', action='store_true', help="Show tuxmake's output" +) args = parser.parse_args() if not (tree := Path(args.directory).resolve()).exists(): @@ -145,9 +138,11 @@ def interrupt_handler(_signum, _frame): ) cfg_str = '+'.join(kconfig) if isinstance(kconfig, list) else kconfig - print(f"I: Building {target_arch} {cfg_str} ({toolchain})... ", - end='\n' if args.verbose else '', - flush=True) + print( + f"I: Building {target_arch} {cfg_str} ({toolchain})... ", + end='\n' if args.verbose else '', + flush=True, + ) if build_dir.exists(): shutil.rmtree(build_dir) @@ -155,7 +150,7 @@ def interrupt_handler(_signum, _frame): # Replace the URL in the configuration string with a simple name, so # that it can be used in a path. - if (match := re.search(r'(https://[^\+]+)', cfg_str)): + if match := re.search(r'(https://[^\+]+)', cfg_str): url = match.groups()[0] if 'alpine' in url: distro = 'alpine' @@ -179,13 +174,15 @@ def interrupt_handler(_signum, _frame): build['kconfig'] = build['kconfig'][0] # Perform the build and report the result - result = tuxmake.build.build(build_dir=build_dir, - output_dir=specific_output_dir, - quiet=(not args.verbose), - runtime=runtime, - tree=tree, - wrapper=wrapper, - **build) + result = tuxmake.build.build( + build_dir=build_dir, + output_dir=specific_output_dir, + quiet=(not args.verbose), + runtime=runtime, + tree=tree, + wrapper=wrapper, + **build, + ) if all(info.passed for info in result.status.values()): print(f"{GREEN}PASS{NORMAL}") diff --git a/scripts/check-logs.py b/scripts/check-logs.py index 6cca7ed9..e0f756cc 100755 --- a/scripts/check-logs.py +++ b/scripts/check-logs.py @@ -17,7 +17,18 @@ import time import urllib.request -from utils import CI_ROOT, get_build, get_image_name, get_requested_llvm_version, print_red, get_cbl_name, show_builds, die, warn, info +from utils import ( + CI_ROOT, + get_build, + get_image_name, + get_requested_llvm_version, + print_red, + get_cbl_name, + show_builds, + die, + warn, + info, +) def _fetch(title, url, dest): @@ -164,9 +175,7 @@ def check_built_config(build): elif line.startswith("# CONFIG_"): name, state = line.split(" ", 2)[1:] if state != "is not set": - warn( - f"Could not parse '{name}' from .config line '{line}'!?" - ) + warn(f"Could not parse '{name}' from .config line '{line}'!?") state = 'n' elif not line.startswith("#"): warn(f"Could not parse .config line '{line}'!?") @@ -202,8 +211,10 @@ def print_clang_info(build): metadata_json = json.loads(metadata_file.read_text(encoding='utf-8')) info("Printing clang-nightly checkout date and hash") parse_cmd = [ - Path(CI_ROOT, "scripts/parse-debian-clang.py"), "--print-info", - "--version-string", metadata_json["compiler"]["version_full"] + Path(CI_ROOT, "scripts/parse-debian-clang.py"), + "--print-info", + "--version-string", + metadata_json["compiler"]["version_full"], ] subprocess.run(parse_cmd, check=True) @@ -238,16 +249,20 @@ def run_boot(build): ] # If we are running a sanitizer build, we should increase the number of # cores and timeout because booting is much slower - if "CONFIG_KASAN=y" in build["kconfig"] or \ - "CONFIG_KCSAN=y" in build["kconfig"] or \ - "CONFIG_UBSAN=y" in build["kconfig"]: + if ( + "CONFIG_KASAN=y" in build["kconfig"] + or "CONFIG_KCSAN=y" in build["kconfig"] + or "CONFIG_UBSAN=y" in build["kconfig"] + ): boot_cmd += ["-s", "4"] if "CONFIG_KASAN=y" in build["kconfig"]: boot_cmd += ["-t", "20m"] else: boot_cmd += ["-t", "10m"] - if "CONFIG_KASAN_KUNIT_TEST=y" in build["kconfig"] or \ - "CONFIG_KCSAN_KUNIT_TEST=y" in build["kconfig"]: + if ( + "CONFIG_KASAN_KUNIT_TEST=y" in build["kconfig"] + or "CONFIG_KCSAN_KUNIT_TEST=y" in build["kconfig"] + ): info("Disabling Oops problem matcher under Sanitizer KUnit build") print("::remove-matcher owner=linux-kernel-oopses::") diff --git a/scripts/check-patches-apply.py b/scripts/check-patches-apply.py index ca09ceed..569c8c86 100755 --- a/scripts/check-patches-apply.py +++ b/scripts/check-patches-apply.py @@ -9,21 +9,19 @@ from tempfile import TemporaryDirectory parser = ArgumentParser( - description='Check that patches apply to their tree before running CI') + description='Check that patches apply to their tree before running CI' +) parser.add_argument( '-p', '--patches-dir', help='Path to patches directory (can be relative or absolute)', required=True, - type=Path) -parser.add_argument('-r', - '--repo', - help='URL to git repository', - required=True) -parser.add_argument('-R', - '--ref', - help='Git reference to apply patches upon', - required=True) + type=Path, +) +parser.add_argument('-r', '--repo', help='URL to git repository', required=True) +parser.add_argument( + '-R', '--ref', help='Git reference to apply patches upon', required=True +) args = parser.parse_args() # If patches directory does not exist, there is nothing to check @@ -43,11 +41,10 @@ with TemporaryDirectory() as workdir: # Fetch the tarball from the repository. This is different for each type of # tree that we support. - if args.repo.startswith( - 'https://git.kernel.org/pub/scm/linux/kernel/git/'): + if args.repo.startswith('https://git.kernel.org/pub/scm/linux/kernel/git/'): # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git -> 'linux' if (base_repo := args.repo.rsplit('/', 1)[1]).endswith('.git'): - base_repo = base_repo[:-len('.git')] + base_repo = base_repo[: -len('.git')] # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git -> # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/snapshot/linux-master.tar.gz @@ -61,15 +58,16 @@ try: print(f"Downloading {tarball_url}...") - tarball = subprocess.run(['curl', '-LSs', tarball_url], - capture_output=True, - check=True).stdout + tarball = subprocess.run( + ['curl', '-LSs', tarball_url], capture_output=True, check=True + ).stdout print(f"Extracting {tarball_url}...") subprocess.run( ['tar', '-C', workdir, f"--strip-components={strip}", '-xzf-'], check=True, - input=tarball) + input=tarball, + ) except subprocess.CalledProcessError: print( 'Downloading or extracting tarball failed! As this may be flakiness on the server end, exiting 0 to have TuxSuite fail later...' @@ -87,7 +85,7 @@ 'GIT_AUTHOR_EMAIL': git_email, 'GIT_COMMITTER_NAME': git_name, 'GIT_COMMITTER_EMAIL': git_email, - } # yapf: disable + } # fmt: off print(f"Creating initial git repository in {workdir}...") subprocess.run(['git', 'init', '-q'], check=True, cwd=workdir) @@ -95,15 +93,18 @@ print(f"Adding files in {workdir}...") subprocess.run(['git', 'add', '.'], check=True, cwd=workdir) - print( - f"Creating initial commit '{args.repo} @ {args.ref}' in {workdir}...") - subprocess.run(['git', 'commit', '-m', f"{args.repo} @ {args.ref}", '-q'], - check=True, - cwd=workdir, - env=git_commit_env_vars) + print(f"Creating initial commit '{args.repo} @ {args.ref}' in {workdir}...") + subprocess.run( + ['git', 'commit', '-m', f"{args.repo} @ {args.ref}", '-q'], + check=True, + cwd=workdir, + env=git_commit_env_vars, + ) print(f"Applying patches in {workdir}...") - subprocess.run(['git', 'quiltimport', '--patches', patches_dir], - check=True, - cwd=workdir, - env=git_commit_env_vars) + subprocess.run( + ['git', 'quiltimport', '--patches', patches_dir], + check=True, + cwd=workdir, + env=git_commit_env_vars, + ) diff --git a/scripts/check-patches.py b/scripts/check-patches.py index c18985f5..05ab1a57 100755 --- a/scripts/check-patches.py +++ b/scripts/check-patches.py @@ -30,7 +30,9 @@ sys.exit(1) # A single line shell command to update a particular series file - update_series_command = f"\t$ ls -1 {folder}/*.patch | sed 's;{folder}/;;' > {folder}/series" + update_series_command = ( + f"\t$ ls -1 {folder}/*.patch | sed 's;{folder}/;;' > {folder}/series" + ) # Next, make sure series file is not missing if not (series := Path(folder, 'series')).exists(): @@ -55,8 +57,7 @@ # the series file (removed from series file, did not remove patch file) for patch in folder.glob('*.patch'): if patch.name not in series_text: - print( - f"{patch.name} not found in {series} but it is in {folder}?\n") + print(f"{patch.name} not found in {series} but it is in {folder}?\n") print('Update the series file:\n') print(update_series_command) sys.exit(1) @@ -77,9 +78,7 @@ sys.exit(1) if patch_folder.exists() and not wf_has_ps_opt: - print( - f"{patch_folder} exists but '--patch-series' not found in {workflow}?\n" - ) + print(f"{patch_folder} exists but '--patch-series' not found in {workflow}?\n") print('Regenerate the TuxSuite and workflow files:\n') print(f"\t$ ./generate.py {tree}\n") print('or remove the patches if they are no longer being used.') diff --git a/scripts/estimate-builds.py b/scripts/estimate-builds.py index da5294e4..7c5d2637 100755 --- a/scripts/estimate-builds.py +++ b/scripts/estimate-builds.py @@ -30,12 +30,13 @@ # Calculate the number of times that a workflow runs in a week based on its # schedule - num_runs = len( - list(croniter.croniter_range(now, week_from_now, tree['schedule']))) + num_runs = len(list(croniter.croniter_range(now, week_from_now, tree['schedule']))) for build in config['builds']: - if tree['git_repo'] == build['git_repo'] and \ - tree['git_ref'] == build['git_ref'] and \ - tree_llvm_ver == build['llvm_version']: + if ( + tree['git_repo'] == build['git_repo'] + and tree['git_ref'] == build['git_ref'] + and tree_llvm_ver == build['llvm_version'] + ): builds_per_tree[tree_name]['total'] += num_runs builds_per_tree[tree_name][tree_llvm_ver] += num_runs @@ -43,9 +44,9 @@ print(f"Total builds per week: {total_builds}") # Sort the list of builds by total number of builds descending -for tree, builds in sorted(builds_per_tree.items(), - key=lambda x: x[1]['total'], - reverse=True): +for tree, builds in sorted( + builds_per_tree.items(), key=lambda x: x[1]['total'], reverse=True +): print(f"\n - tree: {tree}") print(f" total: {builds['total']}") print(' breakdown:') diff --git a/scripts/generate-boot-utils-json.py b/scripts/generate-boot-utils-json.py index 75fe04ca..d6d68faa 100755 --- a/scripts/generate-boot-utils-json.py +++ b/scripts/generate-boot-utils-json.py @@ -12,14 +12,21 @@ repo = Path(__file__).resolve().parents[1] parser = ArgumentParser( - description='Download latest boot-utils release JSON from GitHub API') + description='Download latest boot-utils release JSON from GitHub API' +) parser.add_argument('github_token', help='Value of GITHUB_TOKEN') args = parser.parse_args() curl_cmd = [ - 'curl', '--header', 'Accept: application/vnd.github+json', '--header', - f"Authorization: Bearer {args.github_token}", '--output', - Path(repo, 'boot-utils.json'), '--silent', '--show-error', - 'https://api.github.com/repos/ClangBuiltLinux/boot-utils/releases/latest' + 'curl', + '--header', + 'Accept: application/vnd.github+json', + '--header', + f"Authorization: Bearer {args.github_token}", + '--output', + Path(repo, 'boot-utils.json'), + '--silent', + '--show-error', + 'https://api.github.com/repos/ClangBuiltLinux/boot-utils/releases/latest', ] subprocess.run(curl_cmd, check=True) diff --git a/scripts/markdown-badges.py b/scripts/markdown-badges.py index 8eed6add..236b929c 100755 --- a/scripts/markdown-badges.py +++ b/scripts/markdown-badges.py @@ -23,7 +23,6 @@ def order_to_rank(iterable, item): class ClangVersion(_BaseVersion): - def __init__(self, version): if 'clang-' not in version: raise ValueError(f"Invalid clang version ('{version}') provided?") @@ -35,7 +34,6 @@ def __init__(self, version): class KernelVersion(_BaseVersion): - def __init__(self, version): major = 0 minor = 0 @@ -52,12 +50,12 @@ def __init__(self, version): major = order_to_rank(upstream_trees, version) # LTS releases - if (match := re.search(r'^([\d|\.]+)$', version)): + if match := re.search(r'^([\d|\.]+)$', version): category = 'lts' major, minor = map(int, match.groups()[0].split('.')) # Named maintainer trees - maintainer_trees = ('tip', ) + maintainer_trees = ('tip',) if version in maintainer_trees: category = 'maintainers' major = order_to_rank(maintainer_trees, version) @@ -146,10 +144,16 @@ def svg(workflow): # - Make every column the same width # - Force whitespace to non-breaking (to keep the cell from eliding spaces) # - Replace "-" with non-breaking-dash (to keep the name from wrapping) -print("| | " + " | ".join([ - " " * (max_width - len(col)) + col.replace('-', '‑') - for col in columns -]) + " |") +print( + "| | " + + " | ".join( + [ + " " * (max_width - len(col)) + col.replace('-', '‑') + for col in columns + ] + ) + + " |" +) # Align tree name to the right for readability, and center the badges. print("| ---: |" + " :---: |" * len(columns)) @@ -165,13 +169,14 @@ def svg(workflow): for tree in rows: # Keep names from wrapping. row = tree.replace('-', '‑') - print(f"| {row} | " + " | ".join( - [svg(trees[tree].get(compiler, None)) for compiler in columns]) + " |") + print( + f"| {row} | " + + " | ".join([svg(trees[tree].get(compiler, None)) for compiler in columns]) + + " |" + ) # Output a button for the "Check clang version" workflow, which ensures that # tip of tree LLVM is being updating. This does not need to be a part of the # table above. cv_workflow_url = 'https://github.com/clangbuiltlinux/continuous-integration2/actions/workflows/clang-version.yml' -print( - f"\n[![Check clang version]({cv_workflow_url}/badge.svg)]({cv_workflow_url})" -) +print(f"\n[![Check clang version]({cv_workflow_url}/badge.svg)]({cv_workflow_url})") diff --git a/scripts/parse-debian-clang.py b/scripts/parse-debian-clang.py index eb448384..ec9fee76 100755 --- a/scripts/parse-debian-clang.py +++ b/scripts/parse-debian-clang.py @@ -7,25 +7,29 @@ import subprocess parser = ArgumentParser(description="Parse Debian's clang version") -parser.add_argument('-c', - '--check', - action='store_true', - help='Fail if clang has not been updated in 5 days') -parser.add_argument('-p', - '--print-info', - action='store_true', - help='Print information about clang version') +parser.add_argument( + '-c', + '--check', + action='store_true', + help='Fail if clang has not been updated in 5 days', +) +parser.add_argument( + '-p', + '--print-info', + action='store_true', + help='Print information about clang version', +) parser.add_argument( '-v', '--version-string', - help="Use value as clang version instead of calling 'clang --version'") + help="Use value as clang version instead of calling 'clang --version'", +) args = parser.parse_args() if not (version_string := args.version_string): - clang_version = subprocess.run(['clang', '--version'], - capture_output=True, - check=True, - text=True).stdout + clang_version = subprocess.run( + ['clang', '--version'], capture_output=True, check=True, text=True + ).stdout version_string = clang_version.splitlines()[0] # $ clang-14 --version | head -1 diff --git a/scripts/visualize-builds.py b/scripts/visualize-builds.py index 9026e675..d70bb574 100755 --- a/scripts/visualize-builds.py +++ b/scripts/visualize-builds.py @@ -119,27 +119,29 @@ def output_pretty_table(days, output_file=None): for day, hours in days.items(): col_widths[0] = max(col_widths[0], len(CRON_TO_DAY[day])) for hour in range(HOURS): - col_widths[hour + 1] = max(col_widths[hour + 1], - len(str(len(hours[hour])))) + col_widths[hour + 1] = max(col_widths[hour + 1], len(str(len(hours[hour])))) top_border = "┌" + "┬".join("─" * width for width in col_widths) + "┐" print(top_border, file=output) - header_row = ("│" + - "│".join(f"{cell:^{width}}" - for cell, width in zip(header_cells, col_widths)) + - "│") + header_row = ( + "│" + + "│".join(f"{cell:^{width}}" for cell, width in zip(header_cells, col_widths)) + + "│" + ) print(header_row, file=output) header_sep = "├" + "┼".join("─" * width for width in col_widths) + "┤" print(header_sep, file=output) for day, hours in days.items(): - row_cells = [CRON_TO_DAY[day] - ] + [str(len(hours[hour])) for hour in range(HOURS)] - data_row = ("│" + - "│".join(f"{cell:^{width}}" - for cell, width in zip(row_cells, col_widths)) + - "│") + row_cells = [CRON_TO_DAY[day]] + [ + str(len(hours[hour])) for hour in range(HOURS) + ] + data_row = ( + "│" + + "│".join(f"{cell:^{width}}" for cell, width in zip(row_cells, col_widths)) + + "│" + ) print(data_row, file=output) # Bottom border @@ -149,17 +151,17 @@ def output_pretty_table(days, output_file=None): def parse_args(): parser = argparse.ArgumentParser( - description="Visualize build distribution across days and hours") + description="Visualize build distribution across days and hours" + ) parser.add_argument( "--format", choices=["table", "csv", "pretty"], default="pretty", help="Output format (default: pretty)", ) - parser.add_argument("--output", - "-o", - type=str, - help="Output file (default: stdout)") + parser.add_argument( + "--output", "-o", type=str, help="Output file (default: stdout)" + ) return parser.parse_args() @@ -173,7 +175,8 @@ def main(): if args.output: # pylint: disable-next=consider-using-with output_file = open( # noqa: SIM115 - args.output, "w", encoding="utf-8", newline="") + args.output, "w", encoding="utf-8", newline="" + ) try: if args.format == "csv": diff --git a/utils.py b/utils.py index c1883467..78d9bfdd 100644 --- a/utils.py +++ b/utils.py @@ -36,8 +36,7 @@ def disable_subsys_werror_configs(configs): def get_config_from_generator(): - if not (all_generator_files := sorted( - Path(GENERATOR_ROOT, 'yml').glob('*.yml'))): + if not (all_generator_files := sorted(Path(GENERATOR_ROOT, 'yml').glob('*.yml'))): return die('No generator files could not be found?') generator_pieces = [] @@ -55,11 +54,9 @@ def get_image_name(): arch = os.environ["ARCH"] if arch == "powerpc": subarch = get_cbl_name() - return { - "ppc32": "uImage", - "ppc64": "vmlinux", - "ppc64le": "zImage.epapr" - }[subarch] + return {"ppc32": "uImage", "ppc64": "vmlinux", "ppc64le": "zImage.epapr"}[ + subarch + ] return { "arm": "zImage", "arm64": "Image.gz", @@ -86,7 +83,7 @@ def get_cbl_name(): "aarch64": "arm64", "armv7": "arm32_v7", "riscv64": "riscv", - "x86_64": "x86_64" + "x86_64": "x86_64", } # The URL is https://.../stable..config alpine_arch = base_config.split(".")[-2] @@ -98,7 +95,7 @@ def get_cbl_name(): "i686": "x86", "ppc64le": "ppc64le", "s390x": "s390", - "x86_64": "x86_64" + "x86_64": "x86_64", } # The URL is https://.../kernel--fedora.config fedora_arch = base_config.split("/")[-1].split("-")[1] @@ -111,7 +108,7 @@ def get_cbl_name(): "ppc64le": "ppc64le", "riscv64": "riscv", "s390x": "s390", - "x86_64": "x86_64" + "x86_64": "x86_64", } # The URL is https://...//default suse_arch = base_config.split("/")[-2] @@ -184,9 +181,11 @@ def get_build(): configs = os.environ["CONFIG"].split("+") llvm_version = get_requested_llvm_version() for build in _read_builds(): - if build["target_arch"] == arch and \ - build["toolchain"] == llvm_version and \ - build["kconfig"] == configs: + if ( + build["target_arch"] == arch + and build["toolchain"] == llvm_version + and build["kconfig"] == configs + ): return build print_red("ERROR: Unable to find build") show_builds() @@ -211,8 +210,7 @@ def get_llvm_versions(config, tree_name): def get_patches_hash(tree_name): patches_folder = Path(CI_ROOT, 'patches', tree_name) - patches = sorted( - patches_folder.iterdir()) if patches_folder.exists() else [] + patches = sorted(patches_folder.iterdir()) if patches_folder.exists() else [] text = ''.join(item.read_text(encoding='utf-8') for item in patches) @@ -242,7 +240,7 @@ def update_repository_variable( patches_hash: Optional[str] = None, build_status: Optional[str] = None, other: Optional[Dict[str, str]] = None, - allow_fail_to_pass=False # should a cache entry be allowed to go from 'fail' to 'pass' + allow_fail_to_pass=False, # should a cache entry be allowed to go from 'fail' to 'pass' ): """ Update cache entries. @@ -270,8 +268,11 @@ def update_repository_variable( if patches_hash: cached_value["patches_hash"] = patches_hash if build_status: - if not allow_fail_to_pass and cached_value[ - 'build_status'] == 'fail' and build_status == 'pass': + if ( + not allow_fail_to_pass + and cached_value['build_status'] == 'fail' + and build_status == 'pass' + ): ... else: cached_value["build_status"] = build_status @@ -281,14 +282,10 @@ def update_repository_variable( cached_value = json.dumps(cached_value) - new_value = json.dumps({ - "name": key, - "value": cached_value - }).encode("utf-8") - update_request = urllib.request.Request(url, - data=new_value, - method="PATCH", - headers=http_headers) + new_value = json.dumps({"name": key, "value": cached_value}).encode("utf-8") + update_request = urllib.request.Request( + url, data=new_value, method="PATCH", headers=http_headers + ) urllib.request.urlopen(update_request) # pylint: disable=consider-using-with print(f"""\