From b9f65c6583e668f153b3658be023e19bc7f15ae8 Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Thu, 28 Aug 2025 10:47:09 -0700 Subject: [PATCH 1/8] add pylint check --- eng/tools/azure-sdk-tools/azpysdk/main.py | 2 + eng/tools/azure-sdk-tools/azpysdk/pylint.py | 125 ++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 eng/tools/azure-sdk-tools/azpysdk/pylint.py diff --git a/eng/tools/azure-sdk-tools/azpysdk/main.py b/eng/tools/azure-sdk-tools/azpysdk/main.py index 59f830c205ee..a9ec1cf459cc 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/main.py +++ b/eng/tools/azure-sdk-tools/azpysdk/main.py @@ -16,6 +16,7 @@ from .whl import whl from .import_all import import_all from .mypy import mypy +from .pylint import pylint from ci_tools.scenario import install_into_venv, get_venv_python from ci_tools.functions import get_venv_call @@ -60,6 +61,7 @@ def build_parser() -> argparse.ArgumentParser: whl().register(subparsers, [common]) import_all().register(subparsers, [common]) mypy().register(subparsers, [common]) + pylint().register(subparsers, [common]) return parser diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py new file mode 100644 index 000000000000..975bf80ab880 --- /dev/null +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -0,0 +1,125 @@ +from subprocess import check_call, CalledProcessError +import argparse +import os +import logging +import sys +import tempfile + +from typing import Optional, List +from ci_tools.scenario.generation import create_package_and_install +from subprocess import CalledProcessError, check_call + +from .Check import Check +from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults +from ci_tools.variables import in_ci, set_envvar_defaults +from ci_tools.environment_exclusions import is_check_enabled + +REPO_ROOT = discover_repo_root() +PYLINT_VERSION = "3.2.7" + +class pylint(Check): + def __init__(self) -> None: + super().__init__() + + def register(self, subparsers: "argparse._SubParsersAction", parent_parsers: Optional[List[argparse.ArgumentParser]] = None) -> None: + """Register the pylint check. The pylint check installs pylint and runs pylint against the target package. + """ + parents = parent_parsers or [] + p = subparsers.add_parser("pylint", parents=parents, help="Run the pylint check") + p.set_defaults(func=self.run) + + p.add_argument( + "--next", + default=False, + help="Next version of pylint is being tested.", + required=False, + ) + + def run(self, args: argparse.Namespace) -> int: + """Run the pylint check command.""" + print("Running pylint check...") + + set_envvar_defaults() + targeted = self.get_targeted_directories(args) + + results: List[int] = [] + + for parsed in targeted: + package_dir = parsed.folder + package_name = parsed.name + print(f"Processing {package_name} for pylint check") + + staging_area = tempfile.mkdtemp() + create_package_and_install( + distribution_directory=staging_area, + target_setup=package_dir, + skip_install=False, + cache_dir=None, + work_dir=staging_area, + force_create=False, + package_type="wheel", + pre_download_disabled=False, + ) + + # install dependencies + try: + check_call([ + sys.executable, + "-m", + "pip", + "install", + "azure-pylint-guidelines-checker==0.5.6", + "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + ]) + except CalledProcessError as e: + print("Failed to install dependencies:", e) + return e.returncode + + # install pylint + try: + if (args.next): + # use latest version of pylint + check_call([sys.executable, "-m", "pip", "install", "pylint"]) + else: + check_call([sys.executable, "-m", "pip", "install", f"pylint=={PYLINT_VERSION}"]) + except CalledProcessError as e: + print("Failed to install pylint:", e) + return e.returncode + + top_level_module = parsed.namespace.split(".")[0] + + if in_ci(): + if not is_check_enabled(package_dir, "pylint"): + logging.info( + f"Package {package_name} opts-out of pylint check." + ) + continue + + rcFileLocation = os.path.join(REPO_ROOT, "eng/pylintrc") if args.next else os.path.join(REPO_ROOT, "pylintrc") + + try: + results.append(check_call( + [ + sys.executable, + "-m", + "pylint", + "--rcfile={}".format(rcFileLocation), + "--output-format=parseable", + os.path.join(package_dir, top_level_module), + ] + )) + except CalledProcessError as e: + logging.error( + "{} exited with linting error {}. Please see this link for more information https://aka.ms/azsdk/python/pylint-guide".format(package_name=package_name, e=e.returncode) + ) + if args.next and in_ci(): + from gh_tools.vnext_issue_creator import create_vnext_issue + create_vnext_issue(package_dir, "pylint") + + results.append(e.returncode) + + if args.next and in_ci(): + from gh_tools.vnext_issue_creator import close_vnext_issue + close_vnext_issue(package_name, "pylint") + + return max(results) if results else 0 From 1d5f104eb7a7e06984f64e74ed9d803a0a70130c Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Thu, 28 Aug 2025 10:53:45 -0700 Subject: [PATCH 2/8] minor fix --- eng/tools/azure-sdk-tools/azpysdk/pylint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py index 975bf80ab880..eeb3bfb92096 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pylint.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -110,7 +110,7 @@ def run(self, args: argparse.Namespace) -> int: )) except CalledProcessError as e: logging.error( - "{} exited with linting error {}. Please see this link for more information https://aka.ms/azsdk/python/pylint-guide".format(package_name=package_name, e=e.returncode) + "{} exited with linting error {}. Please see this link for more information https://aka.ms/azsdk/python/pylint-guide".format(package_name, e.returncode) ) if args.next and in_ci(): from gh_tools.vnext_issue_creator import create_vnext_issue From 16bcc86e9cdc78e5d3951cd08cff0a314f2b8f09 Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:15:39 -0700 Subject: [PATCH 3/8] remove creating new package installation --- eng/tools/azure-sdk-tools/azpysdk/mypy.py | 12 ------------ eng/tools/azure-sdk-tools/azpysdk/pylint.py | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/mypy.py b/eng/tools/azure-sdk-tools/azpysdk/mypy.py index 7d0e0ae4256c..1a619e021182 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/mypy.py +++ b/eng/tools/azure-sdk-tools/azpysdk/mypy.py @@ -54,18 +54,6 @@ def run(self, args: argparse.Namespace) -> int: package_name = parsed.name print(f"Processing {package_name} for mypy check") - staging_area = tempfile.mkdtemp() - create_package_and_install( - distribution_directory=staging_area, - target_setup=package_dir, - skip_install=False, - cache_dir=None, - work_dir=staging_area, - force_create=False, - package_type="wheel", - pre_download_disabled=False, - ) - # install mypy try: if (args.next): diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py index eeb3bfb92096..3be1cf4db7c9 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pylint.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -49,18 +49,6 @@ def run(self, args: argparse.Namespace) -> int: package_name = parsed.name print(f"Processing {package_name} for pylint check") - staging_area = tempfile.mkdtemp() - create_package_and_install( - distribution_directory=staging_area, - target_setup=package_dir, - skip_install=False, - cache_dir=None, - work_dir=staging_area, - force_create=False, - package_type="wheel", - pre_download_disabled=False, - ) - # install dependencies try: check_call([ From 321e1b320307725f9ff462e9c94fdab1d6bc631b Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:24:02 -0700 Subject: [PATCH 4/8] clean imports --- eng/tools/azure-sdk-tools/azpysdk/pylint.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py index 3be1cf4db7c9..1b0656952ca9 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pylint.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -1,17 +1,13 @@ -from subprocess import check_call, CalledProcessError import argparse import os import logging import sys -import tempfile from typing import Optional, List -from ci_tools.scenario.generation import create_package_and_install from subprocess import CalledProcessError, check_call from .Check import Check -from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults -from ci_tools.variables import in_ci, set_envvar_defaults +from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults, in_ci, set_envvar_defaults from ci_tools.environment_exclusions import is_check_enabled REPO_ROOT = discover_repo_root() @@ -65,7 +61,7 @@ def run(self, args: argparse.Namespace) -> int: # install pylint try: - if (args.next): + if args.next: # use latest version of pylint check_call([sys.executable, "-m", "pip", "install", "pylint"]) else: From cec631037bbea4d7a47119cf0d0205d080f2c1f8 Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:28:27 -0700 Subject: [PATCH 5/8] clean mypy imports --- eng/tools/azure-sdk-tools/azpysdk/mypy.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/mypy.py b/eng/tools/azure-sdk-tools/azpysdk/mypy.py index 1a619e021182..ea2ecd43a47d 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/mypy.py +++ b/eng/tools/azure-sdk-tools/azpysdk/mypy.py @@ -2,15 +2,11 @@ import os import sys import logging -import tempfile from typing import Optional, List from subprocess import CalledProcessError, check_call from .Check import Check -from ci_tools.parsing import ParsedSetup -from ci_tools.functions import discover_targeted_packages -from ci_tools.scenario.generation import create_package_and_install from ci_tools.variables import in_ci, set_envvar_defaults from ci_tools.environment_exclusions import ( is_check_enabled, is_typing_ignored From 9e0d5661797098d89539d56c86775b2e463f925f Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Fri, 29 Aug 2025 10:30:43 -0700 Subject: [PATCH 6/8] merge updates --- eng/tools/azure-sdk-tools/azpysdk/mypy.py | 11 ----------- eng/tools/azure-sdk-tools/azpysdk/pylint.py | 9 +++++---- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/mypy.py b/eng/tools/azure-sdk-tools/azpysdk/mypy.py index ea8f04d462c3..a8e344e60360 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/mypy.py +++ b/eng/tools/azure-sdk-tools/azpysdk/mypy.py @@ -53,17 +53,6 @@ def run(self, args: argparse.Namespace) -> int: package_name = parsed.name executable, staging_directory = self.get_executable(args.isolate, args.command, sys.executable, package_dir) print(f"Processing {package_name} for mypy check") - create_package_and_install( - distribution_directory=staging_directory, - target_setup=package_dir, - skip_install=False, - cache_dir=None, - work_dir=staging_directory, - force_create=False, - package_type="wheel", - pre_download_disabled=False, - python_executable=executable - ) # install mypy try: diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py index 1b0656952ca9..24dd6c05ea20 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pylint.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -43,12 +43,13 @@ def run(self, args: argparse.Namespace) -> int: for parsed in targeted: package_dir = parsed.folder package_name = parsed.name + executable, staging_directory = self.get_executable(args.isolate, args.command, sys.executable, package_dir) print(f"Processing {package_name} for pylint check") # install dependencies try: check_call([ - sys.executable, + executable, "-m", "pip", "install", @@ -63,9 +64,9 @@ def run(self, args: argparse.Namespace) -> int: try: if args.next: # use latest version of pylint - check_call([sys.executable, "-m", "pip", "install", "pylint"]) + check_call([executable, "-m", "pip", "install", "pylint"]) else: - check_call([sys.executable, "-m", "pip", "install", f"pylint=={PYLINT_VERSION}"]) + check_call([executable, "-m", "pip", "install", f"pylint=={PYLINT_VERSION}"]) except CalledProcessError as e: print("Failed to install pylint:", e) return e.returncode @@ -84,7 +85,7 @@ def run(self, args: argparse.Namespace) -> int: try: results.append(check_call( [ - sys.executable, + executable, "-m", "pylint", "--rcfile={}".format(rcFileLocation), From 819c5d07b75a35bcc77ca008048b4cd6e9885656 Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Fri, 29 Aug 2025 11:19:39 -0700 Subject: [PATCH 7/8] merge updates and remove package install --- eng/tools/azure-sdk-tools/azpysdk/mypy.py | 12 ------------ eng/tools/azure-sdk-tools/azpysdk/pylint.py | 14 +++++++------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/mypy.py b/eng/tools/azure-sdk-tools/azpysdk/mypy.py index eef2960571ce..b074eadfbf50 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/mypy.py +++ b/eng/tools/azure-sdk-tools/azpysdk/mypy.py @@ -53,18 +53,6 @@ def run(self, args: argparse.Namespace) -> int: executable, staging_directory = self.get_executable(args.isolate, args.command, sys.executable, package_dir) logger.info(f"Processing {package_name} for mypy check") - - create_package_and_install( - distribution_directory=staging_directory, - target_setup=package_dir, - skip_install=False, - cache_dir=None, - work_dir=staging_directory, - force_create=False, - package_type="wheel", - pre_download_disabled=False, - python_executable=executable - ) # install mypy try: diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py index 24dd6c05ea20..06579b63e3f8 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pylint.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -1,6 +1,5 @@ import argparse import os -import logging import sys from typing import Optional, List @@ -9,6 +8,7 @@ from .Check import Check from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults, in_ci, set_envvar_defaults from ci_tools.environment_exclusions import is_check_enabled +from ci_tools.logging import logger REPO_ROOT = discover_repo_root() PYLINT_VERSION = "3.2.7" @@ -33,7 +33,7 @@ def register(self, subparsers: "argparse._SubParsersAction", parent_parsers: Opt def run(self, args: argparse.Namespace) -> int: """Run the pylint check command.""" - print("Running pylint check...") + logger.info("Running pylint check...") set_envvar_defaults() targeted = self.get_targeted_directories(args) @@ -44,7 +44,7 @@ def run(self, args: argparse.Namespace) -> int: package_dir = parsed.folder package_name = parsed.name executable, staging_directory = self.get_executable(args.isolate, args.command, sys.executable, package_dir) - print(f"Processing {package_name} for pylint check") + logger.info(f"Processing {package_name} for pylint check") # install dependencies try: @@ -57,7 +57,7 @@ def run(self, args: argparse.Namespace) -> int: "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" ]) except CalledProcessError as e: - print("Failed to install dependencies:", e) + logger.error("Failed to install dependencies:", e) return e.returncode # install pylint @@ -68,14 +68,14 @@ def run(self, args: argparse.Namespace) -> int: else: check_call([executable, "-m", "pip", "install", f"pylint=={PYLINT_VERSION}"]) except CalledProcessError as e: - print("Failed to install pylint:", e) + logger.error("Failed to install pylint:", e) return e.returncode top_level_module = parsed.namespace.split(".")[0] if in_ci(): if not is_check_enabled(package_dir, "pylint"): - logging.info( + logger.info( f"Package {package_name} opts-out of pylint check." ) continue @@ -94,7 +94,7 @@ def run(self, args: argparse.Namespace) -> int: ] )) except CalledProcessError as e: - logging.error( + logger.error( "{} exited with linting error {}. Please see this link for more information https://aka.ms/azsdk/python/pylint-guide".format(package_name, e.returncode) ) if args.next and in_ci(): From 85ceb864ae97e2ce5c4ea6513463fb8e1ae2461d Mon Sep 17 00:00:00 2001 From: jenny <63012604+JennyPng@users.noreply.github.com> Date: Fri, 29 Aug 2025 12:04:09 -0700 Subject: [PATCH 8/8] use pip_install --- eng/tools/azure-sdk-tools/azpysdk/pylint.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py index 06579b63e3f8..16c9a7a58965 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pylint.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -6,6 +6,7 @@ from subprocess import CalledProcessError, check_call from .Check import Check +from ci_tools.functions import pip_install from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults, in_ci, set_envvar_defaults from ci_tools.environment_exclusions import is_check_enabled from ci_tools.logging import logger @@ -48,14 +49,9 @@ def run(self, args: argparse.Namespace) -> int: # install dependencies try: - check_call([ - executable, - "-m", - "pip", - "install", - "azure-pylint-guidelines-checker==0.5.6", - "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" - ]) + pip_install([ + "azure-pylint-guidelines-checker==0.5.6", "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + ], True, executable, package_dir) except CalledProcessError as e: logger.error("Failed to install dependencies:", e) return e.returncode @@ -64,9 +60,9 @@ def run(self, args: argparse.Namespace) -> int: try: if args.next: # use latest version of pylint - check_call([executable, "-m", "pip", "install", "pylint"]) + pip_install(["pylint"], True, executable, package_dir) else: - check_call([executable, "-m", "pip", "install", f"pylint=={PYLINT_VERSION}"]) + pip_install([f"pylint=={PYLINT_VERSION}"], True, executable, package_dir) except CalledProcessError as e: logger.error("Failed to install pylint:", e) return e.returncode