diff --git a/eng/tools/azure-sdk-tools/README.md b/eng/tools/azure-sdk-tools/README.md index ce9ab4f20de3..99be5708c28f 100644 --- a/eng/tools/azure-sdk-tools/README.md +++ b/eng/tools/azure-sdk-tools/README.md @@ -201,16 +201,14 @@ class my_check(Check): """This is the recommended """ set_envvar_defaults() - if args.target == ".": - targeted = [os.getcwd()] - else: - target_dir = os.getcwd() - targeted = discover_targeted_packages(args.target, target_dir) + targeted = self.get_targeted_directories(args) + results: List[int] = [] - for pkg in targeted: - parsed = ParsedSetup.from_path(pkg) - print(f"Processing {pkg.name} for my_check") + for parsed in targeted: + pkg_dir = parsed.folder + pkg_name = parsed.name + print(f"Processing {pkg_name} for my_check") ``` - Once the new check has been created, register it in `azpysdk/main.py` on line 58. This will likely be automated by decorator in the near future but this is out how it should be done for now. diff --git a/eng/tools/azure-sdk-tools/azpysdk/Check.py b/eng/tools/azure-sdk-tools/azpysdk/Check.py index 1cb90b4478f2..cd18731eed5b 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/Check.py +++ b/eng/tools/azure-sdk-tools/azpysdk/Check.py @@ -1,6 +1,10 @@ import abc +import os import argparse +import traceback from typing import Sequence, Optional, List, Any +from ci_tools.parsing import ParsedSetup +from ci_tools.functions import discover_targeted_packages class Check(abc.ABC): """ @@ -28,4 +32,30 @@ def run(self, args: argparse.Namespace) -> int: Subclasses can override this to perform the actual work. """ - return 0 \ No newline at end of file + return 0 + + def get_targeted_directories(self, args: argparse.Namespace) -> List[ParsedSetup]: + """ + Get the directories that are targeted for the check. + """ + targeted: List[ParsedSetup] = [] + targeted_dir = os.getcwd() + + if args.target == ".": + try: + targeted.append(ParsedSetup.from_path(targeted_dir)) + except Exception as e: + print("Error: Current directory does not appear to be a Python package (no setup.py or setup.cfg found). Remove '.' argument to run on child directories.") + print(f"Exception: {e}") + return [] + else: + targeted_packages = discover_targeted_packages(args.target, targeted_dir) + for pkg in targeted_packages: + try: + targeted.append(ParsedSetup.from_path(pkg)) + except Exception as e: + print(f"Unable to parse {pkg} as a Python package. Dumping exception detail and skipping.") + print(f"Exception: {e}") + print(traceback.format_exc()) + + return targeted diff --git a/eng/tools/azure-sdk-tools/azpysdk/import_all.py b/eng/tools/azure-sdk-tools/azpysdk/import_all.py index 2da92970dfd2..0dd4ea27cefd 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/import_all.py +++ b/eng/tools/azure-sdk-tools/azpysdk/import_all.py @@ -44,20 +44,16 @@ def run(self, args: argparse.Namespace) -> int: """Run the import_all check command.""" print("Running import_all check in isolated venv...") - # this is common. we should have an abstraction layer for this somehow - if args.target == ".": - targeted = [os.getcwd()] - else: - target_dir = os.getcwd() - targeted = discover_targeted_packages(args.target, target_dir) + targeted = self.get_targeted_directories(args) # {[tox]pip_command} freeze # python {repository_root}/eng/tox/import_all.py -t {tox_root} outcomes: List[int] = [] - for pkg in targeted: - parsed = ParsedSetup.from_path(pkg) + for parsed in targeted: + pkg = parsed.folder + staging_area = tempfile.mkdtemp() create_package_and_install( diff --git a/eng/tools/azure-sdk-tools/azpysdk/main.py b/eng/tools/azure-sdk-tools/azpysdk/main.py index 5b89c7f65d68..59f830c205ee 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/main.py +++ b/eng/tools/azure-sdk-tools/azpysdk/main.py @@ -15,6 +15,7 @@ from .whl import whl from .import_all import import_all +from .mypy import mypy from ci_tools.scenario import install_into_venv, get_venv_python from ci_tools.functions import get_venv_call @@ -42,8 +43,8 @@ def build_parser() -> argparse.ArgumentParser: common.add_argument( "target", nargs="?", - default=".", - help="Glob pattern for packages. Defaults to '.', but will match patterns below CWD if a value is provided." + default="**", + help="Glob pattern for packages. Defaults to '**', but will match patterns below CWD if a value is provided." ) # allow --isolate to be specified after the subcommand as well common.add_argument( @@ -58,6 +59,7 @@ def build_parser() -> argparse.ArgumentParser: # register our checks with the common params as their parent whl().register(subparsers, [common]) import_all().register(subparsers, [common]) + mypy().register(subparsers, [common]) return parser diff --git a/eng/tools/azure-sdk-tools/azpysdk/mypy.py b/eng/tools/azure-sdk-tools/azpysdk/mypy.py new file mode 100644 index 000000000000..7d0e0ae4256c --- /dev/null +++ b/eng/tools/azure-sdk-tools/azpysdk/mypy.py @@ -0,0 +1,150 @@ +import argparse +import os +import sys +import logging +import tempfile + +from typing import Optional, List +from subprocess import CalledProcessError, check_call + +from .Check import Check +from ci_tools.parsing import ParsedSetup +from ci_tools.functions import discover_targeted_packages +from ci_tools.scenario.generation import create_package_and_install +from ci_tools.variables import in_ci, set_envvar_defaults +from ci_tools.environment_exclusions import ( + is_check_enabled, is_typing_ignored +) + +logging.getLogger().setLevel(logging.INFO) + +PYTHON_VERSION = "3.9" +MYPY_VERSION = "1.14.1" + +class mypy(Check): + def __init__(self) -> None: + super().__init__() + + def register(self, subparsers: "argparse._SubParsersAction", parent_parsers: Optional[List[argparse.ArgumentParser]] = None) -> None: + """Register the `mypy` check. The mypy check installs mypy and runs mypy against the target package. + """ + parents = parent_parsers or [] + p = subparsers.add_parser("mypy", parents=parents, help="Run the mypy check") + p.set_defaults(func=self.run) + + p.add_argument( + "--next", + default=False, + help="Next version of mypy is being tested", + required=False + ) + + def run(self, args: argparse.Namespace) -> int: + """Run the mypy check command.""" + print("Running mypy check in isolated venv...") + + set_envvar_defaults() + + targeted = self.get_targeted_directories(args) + + results: List[int] = [] + + for parsed in targeted: + package_dir = parsed.folder + package_name = parsed.name + print(f"Processing {package_name} for mypy check") + + staging_area = tempfile.mkdtemp() + create_package_and_install( + distribution_directory=staging_area, + target_setup=package_dir, + skip_install=False, + cache_dir=None, + work_dir=staging_area, + force_create=False, + package_type="wheel", + pre_download_disabled=False, + ) + + # install mypy + try: + if (args.next): + # use latest version of mypy + check_call([sys.executable, "-m", "pip", "install", "mypy"]) + else: + check_call([sys.executable, "-m", "pip", "install", f"mypy=={MYPY_VERSION}"]) + except CalledProcessError as e: + print("Failed to install mypy:", e) + return e.returncode + + logging.info(f"Running mypy against {package_name}") + + if not args.next and in_ci(): + if not is_check_enabled(package_dir, "mypy", True) or is_typing_ignored(package_name): + logging.info( + f"Package {package_name} opts-out of mypy check. See https://aka.ms/python/typing-guide for information." + ) + continue + + top_level_module = parsed.namespace.split(".")[0] + + commands = [ + sys.executable, + "-m", + "mypy", + "--python-version", + PYTHON_VERSION, + "--show-error-codes", + "--ignore-missing-imports", + ] + src_code = [*commands, os.path.join(package_dir, top_level_module)] + src_code_error = None + sample_code_error = None + try: + logging.info( + f"Running mypy commands on src code: {src_code}" + ) + results.append(check_call(src_code)) + logging.info("Verified mypy, no issues found") + except CalledProcessError as src_error: + src_code_error = src_error + results.append(src_error.returncode) + + if not args.next and in_ci() and not is_check_enabled(package_dir, "type_check_samples", True): + logging.info( + f"Package {package_name} opts-out of mypy check on samples." + ) + continue + else: + # check if sample dirs exists, if not, skip sample code check + samples = os.path.exists(os.path.join(package_dir, "samples")) + generated_samples = os.path.exists(os.path.join(package_dir, "generated_samples")) + if not samples and not generated_samples: + logging.info( + f"Package {package_name} does not have a samples directory." + ) + else: + sample_code = [ + *commands, + "--check-untyped-defs", + "--follow-imports=silent", + os.path.join(package_dir, "samples" if samples else "generated_samples"), + ] + try: + logging.info( + f"Running mypy commands on sample code: {sample_code}" + ) + results.append(check_call(sample_code)) + except CalledProcessError as sample_error: + sample_code_error = sample_error + results.append(sample_error.returncode) + + if args.next and in_ci() and not is_typing_ignored(package_name): + from gh_tools.vnext_issue_creator import create_vnext_issue, close_vnext_issue + if src_code_error or sample_code_error: + create_vnext_issue(package_dir, "mypy") + else: + close_vnext_issue(package_name, "mypy") + + return max(results) if results else 0 + \ No newline at end of file diff --git a/eng/tools/azure-sdk-tools/azpysdk/whl.py b/eng/tools/azure-sdk-tools/azpysdk/whl.py index 0e879f049c37..b6d9bc4f8f93 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/whl.py +++ b/eng/tools/azure-sdk-tools/azpysdk/whl.py @@ -33,15 +33,12 @@ def run(self, args: argparse.Namespace) -> int: set_envvar_defaults() - if args.target == ".": - targeted = [os.getcwd()] - else: - target_dir = os.getcwd() - targeted = discover_targeted_packages(args.target, target_dir) + targeted = self.get_targeted_directories(args) + results: List[int] = [] - for pkg in targeted: - parsed = ParsedSetup.from_path(pkg) + for parsed in targeted: + pkg = parsed.folder dev_requirements = os.path.join(pkg, "dev_requirements.txt")