diff --git a/eng/pipelines/templates/steps/run_bandit.yml b/eng/pipelines/templates/steps/run_bandit.yml
index 0b02f7c10a7c..246bc32b2b2a 100644
--- a/eng/pipelines/templates/steps/run_bandit.yml
+++ b/eng/pipelines/templates/steps/run_bandit.yml
@@ -6,11 +6,8 @@ parameters:
# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
steps:
- - script: |
- python -m pip install -r eng/ci_tools.txt
- displayName: 'Prep Environment'
- condition: and(succeededOrFailed(), ne(variables['Skip.Bandit'],'true'))
-
+ # this step should run on python 3.9.
+ # a previous step should have installed eng/ci_tools.txt to active venv
- task: PythonScript@0
displayName: 'Run Bandit'
inputs:
diff --git a/eng/pipelines/templates/steps/run_black.yml b/eng/pipelines/templates/steps/run_black.yml
index da531b919c18..ce7f9f58ca89 100644
--- a/eng/pipelines/templates/steps/run_black.yml
+++ b/eng/pipelines/templates/steps/run_black.yml
@@ -4,17 +4,8 @@ parameters:
AdditionalTestArgs: ''
steps:
- - task: UsePythonVersion@0
- displayName: 'Use Python 3.9'
- inputs:
- versionSpec: '3.9'
- condition: succeededOrFailed()
-
- - script: |
- python -m pip install -r eng/ci_tools.txt
- displayName: 'Prep Environment'
- condition: succeededOrFailed()
-
+ # this step should run on python 3.9.
+ # a previous step should have installed eng/ci_tools.txt to active venv
- task: PythonScript@0
displayName: 'Run Black'
inputs:
diff --git a/eng/pipelines/templates/steps/run_mypy.yml b/eng/pipelines/templates/steps/run_mypy.yml
index 5da722a51726..1e95cb799946 100644
--- a/eng/pipelines/templates/steps/run_mypy.yml
+++ b/eng/pipelines/templates/steps/run_mypy.yml
@@ -2,32 +2,24 @@ parameters:
BuildTargetingString: 'azure-*'
ServiceDirectory: ''
TestMarkArgument: ''
- EnvVars: {}
AdditionalTestArgs: ''
# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
steps:
- - task: UsePythonVersion@0
- displayName: 'Use Python 3.9'
- inputs:
- versionSpec: '3.9'
- condition: and(succeededOrFailed(), ne(variables['Skip.MyPy'],'true'))
-
- - script: |
- python -m pip install -r eng/ci_tools.txt
- displayName: 'Prep Environment'
- condition: and(succeededOrFailed(), ne(variables['Skip.MyPy'],'true'))
-
+ # this step should run on python 3.9.
+ # a previous step should have installed eng/ci_tools.txt to active venv
- task: PythonScript@0
displayName: 'Run MyPy'
inputs:
- scriptPath: 'scripts/devops_tasks/dispatch_tox.py'
+ scriptPath: 'eng/scripts/dispatch_checks.py'
arguments: >-
"$(TargetingString)"
- --mark_arg="${{ parameters.TestMarkArgument }}"
--service="${{ parameters.ServiceDirectory }}"
- --toxenv="mypy"
+ --checks="mypy"
--disablecov
${{ parameters.AdditionalTestArgs }}
- env: ${{ parameters.EnvVars }}
+ env:
+ TOX_PIP_IMPL: "uv"
+ VIRTUAL_ENV: ""
+ PYTHONHOME: ""
condition: and(succeededOrFailed(), ne(variables['Skip.MyPy'],'true'))
diff --git a/eng/pipelines/templates/steps/run_pylint.yml b/eng/pipelines/templates/steps/run_pylint.yml
index afb489b3ff84..02215027c860 100644
--- a/eng/pipelines/templates/steps/run_pylint.yml
+++ b/eng/pipelines/templates/steps/run_pylint.yml
@@ -6,17 +6,8 @@ parameters:
# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
steps:
- - task: UsePythonVersion@0
- displayName: 'Use Python 3.9'
- inputs:
- versionSpec: '3.9'
- condition: and(succeededOrFailed(), ne(variables['Skip.Pylint'],'true'))
-
- - script: |
- python -m pip install -r eng/ci_tools.txt
- displayName: 'Prep Environment'
- condition: and(succeededOrFailed(), ne(variables['Skip.Pylint'],'true'))
-
+ # this step should run on python 3.9.
+ # a previous step should have installed eng/ci_tools.txt to active venv
- task: PythonScript@0
displayName: 'Run Pylint'
inputs:
diff --git a/eng/pipelines/templates/steps/run_pyright.yml b/eng/pipelines/templates/steps/run_pyright.yml
index b5a28ece08a4..1a26ed13ea35 100644
--- a/eng/pipelines/templates/steps/run_pyright.yml
+++ b/eng/pipelines/templates/steps/run_pyright.yml
@@ -6,17 +6,8 @@ parameters:
# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
steps:
- - task: UsePythonVersion@0
- displayName: 'Use Python 3.9'
- inputs:
- versionSpec: '3.9'
- condition: and(succeededOrFailed(), or(ne(variables['Skip.Pyright'],'true'), ne(variables['Skip.Verifytypes'],'true')))
-
- - script: |
- python -m pip install -r eng/ci_tools.txt
- displayName: 'Prep Environment'
- condition: and(succeededOrFailed(), or(ne(variables['Skip.Pyright'],'true'), ne(variables['Skip.Verifytypes'],'true')))
-
+ # this step should run on python 3.9.
+ # a previous step should have installed eng/ci_tools.txt to active venv
- task: PythonScript@0
displayName: 'Run Pyright'
inputs:
diff --git a/eng/pipelines/templates/steps/update_snippet.yml b/eng/pipelines/templates/steps/update_snippet.yml
index be991e2f16d1..9aab04508cbb 100644
--- a/eng/pipelines/templates/steps/update_snippet.yml
+++ b/eng/pipelines/templates/steps/update_snippet.yml
@@ -4,12 +4,8 @@ parameters:
default: ''
steps:
- - task: UsePythonVersion@0
- displayName: 'Use Python 3.9'
- inputs:
- versionSpec: '3.9'
- condition: succeededOrFailed()
-
+ # this step should run on python 3.9.
+ # a previous step should have installed eng/ci_tools.txt to active venv
- pwsh: |
$failed = $false
diff --git a/eng/scripts/dispatch_checks.py b/eng/scripts/dispatch_checks.py
new file mode 100644
index 000000000000..1a9435809bad
--- /dev/null
+++ b/eng/scripts/dispatch_checks.py
@@ -0,0 +1,338 @@
+import argparse
+import asyncio
+import os
+import sys
+import time
+import signal
+from dataclasses import dataclass
+from typing import List
+
+from ci_tools.functions import discover_targeted_packages
+from ci_tools.variables import in_ci
+from ci_tools.scenario.generation import build_whl_for_req
+from ci_tools.logging import configure_logging, logger
+from ci_tools.environment_exclusions import is_check_enabled
+
+root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+
+@dataclass
+class CheckResult:
+ package: str
+ check: str
+ exit_code: int
+ duration: float
+ stdout: str
+ stderr: str
+
+
+async def run_check(semaphore: asyncio.Semaphore, package: str, check: str, base_args: List[str], idx: int, total: int) -> CheckResult:
+ """Run a single check (subprocess) within a concurrency semaphore, capturing output and timing.
+
+ :param semaphore: Concurrency limiter used to bound concurrent checks.
+ :type semaphore: asyncio.Semaphore
+ :param package: Absolute path to the package directory used as the subprocess cwd.
+ :type package: str
+ :param check: The check (subcommand) name for the azpysdk CLI to invoke.
+ :type check: str
+ :param base_args: Common argument list prefix (e.g. ``[sys.executable, "-m", "azpysdk.main"]``).
+ :type base_args: List[str]
+ :param idx: Sequence number for logging (1-based index of this task).
+ :type idx: int
+ :param total: Total number of tasks (used for logging progress).
+ :type total: int
+ :returns: A :class:`CheckResult` describing exit code, duration and captured output.
+ :rtype: CheckResult
+ """
+ async with semaphore:
+ start = time.time()
+ cmd = base_args + [check, "--isolate", package]
+ logger.info(f"[START {idx}/{total}] {check} :: {package}\nCMD: {' '.join(cmd)}")
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ *cmd,
+ cwd=package,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ except Exception as ex: # subprocess failed to launch
+ logger.error(f"Failed to start check {check} for {package}: {ex}")
+ return CheckResult(package, check, 127, 0.0, "", str(ex))
+
+ stdout_b, stderr_b = await proc.communicate()
+ duration = time.time() - start
+ stdout = stdout_b.decode(errors="replace")
+ stderr = stderr_b.decode(errors="replace")
+ exit_code = proc.returncode or 0
+ status = "OK" if exit_code == 0 else f"FAIL({exit_code})"
+ logger.info(f"[END {idx}/{total}] {check} :: {package} -> {status} in {duration:.2f}s")
+ # Print captured output after completion to avoid interleaving
+ header = f"===== OUTPUT: {check} :: {package} (exit {exit_code}) ====="
+ trailer = "=" * len(header)
+ if stdout:
+ print(header)
+ print(stdout.rstrip())
+ print(trailer)
+ if stderr:
+ print(header.replace('OUTPUT', 'STDERR'))
+ print(stderr.rstrip())
+ print(trailer)
+ return CheckResult(package, check, exit_code, duration, stdout, stderr)
+
+
+def summarize(results: List[CheckResult]) -> int:
+ """Print a compact summary table and return the worst exit code.
+
+ The function prints a human-readable table to stdout showing package, check, status and
+ duration. It returns the highest (worst) exit code from the provided results.
+
+ :param results: List of :class:`CheckResult` objects to summarize.
+ :type results: List[CheckResult]
+ :returns: The maximum exit code found in ``results`` (0 if all passed).
+ :rtype: int
+ """
+ # Compute column widths
+ pkg_w = max((len(r.package) for r in results), default=7)
+ chk_w = max((len(r.check) for r in results), default=5)
+ header = f"{'PACKAGE'.ljust(pkg_w)} {'CHECK'.ljust(chk_w)} STATUS DURATION(s)"
+ print("\n=== SUMMARY ===")
+ print(header)
+ print("-" * len(header))
+ for r in sorted(results, key=lambda x: (x.exit_code != 0, x.package, x.check)):
+ status = "OK" if r.exit_code == 0 else f"FAIL({r.exit_code})"
+ print(f"{r.package.ljust(pkg_w)} {r.check.ljust(chk_w)} {status.ljust(8)} {r.duration:>10.2f}")
+ worst = max((r.exit_code for r in results), default=0)
+ failed = [r for r in results if r.exit_code != 0]
+ print(f"\nTotal checks: {len(results)} | Failed: {len(failed)} | Worst exit code: {worst}")
+ return worst
+
+
+async def run_all_checks(packages, checks, max_parallel):
+ """Run all checks for all packages concurrently and return the worst exit code.
+
+ :param packages: Iterable of package paths to run checks against.
+ :type packages: Iterable[str]
+ :param checks: List of check names to execute for each package.
+ :type checks: List[str]
+ :param max_parallel: Maximum number of concurrent checks to run.
+ :type max_parallel: int
+ :returns: The worst exit code from all checks (0 if all passed).
+ :rtype: int
+ """
+ base_args = [sys.executable, "-m", "azpysdk.main"]
+ tasks = []
+ semaphore = asyncio.Semaphore(max_parallel)
+ combos = [(p, c) for p in packages for c in checks]
+ total = len(combos)
+ for idx, (package, check) in enumerate(combos, start=1):
+ if not is_check_enabled(package, check):
+ logger.warning(f"Skipping disabled check {check} ({idx}/{total}) for package {package}")
+ continue
+ tasks.append(asyncio.create_task(run_check(semaphore, package, check, base_args, idx, total)))
+
+ # Handle Ctrl+C gracefully
+ pending = set(tasks)
+ try:
+ results = await asyncio.gather(*tasks, return_exceptions=True)
+ except KeyboardInterrupt:
+ logger.warning("KeyboardInterrupt received. Cancelling running checks...")
+ for t in pending:
+ t.cancel()
+ raise
+ # Normalize exceptions
+ norm_results: List[CheckResult] = []
+ for res, (package, check) in zip(results, combos):
+ if isinstance(res, CheckResult):
+ norm_results.append(res)
+ elif isinstance(res, Exception):
+ norm_results.append(CheckResult(package, check, 99, 0.0, "", str(res)))
+ else:
+ norm_results.append(CheckResult(package, check, 98, 0.0, "", f"Unknown result type: {res}"))
+ return summarize(norm_results)
+
+
+def configure_interrupt_handling():
+ """Install a SIGINT handler that triggers graceful shutdown.
+
+ Registers a handler for SIGINT which raises :class:`KeyboardInterrupt` to allow
+ the asyncio event loop to cancel tasks and subprocesses cleanly. On platforms or
+ contexts where ``signal.signal`` is not supported (for example non-main threads),
+ registration is skipped silently.
+
+ :returns: None
+ :rtype: None
+ """
+
+ # Ensure that a SIGINT propagates to asyncio tasks & subprocesses
+ def handler(signum, frame):
+ """Signal handler for SIGINT.
+
+ Logs receipt of the signal and raises :class:`KeyboardInterrupt` to trigger
+ graceful shutdown of asyncio tasks and subprocesses.
+
+ :param signum: The numeric signal received (e.g. ``signal.SIGINT``).
+ :type signum: int
+ :param frame: Current stack frame when the signal was received (may be ``None``).
+ :type frame: object
+ :raises KeyboardInterrupt: Always raised to signal shutdown.
+ """
+ logger.warning(f"Received signal {signum}. Attempting graceful shutdown...")
+ # Let asyncio loop raise KeyboardInterrupt
+ raise KeyboardInterrupt
+
+ try:
+ signal.signal(signal.SIGINT, handler)
+ except (ValueError, AttributeError): # not supported on some platforms/threads
+ pass
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ description="""
+This script is the single point for all checks invoked by CI within this repo. It works in two phases.
+ 1. Identify which packages in the repo are in scope for this script invocation, based on a glob string and a service directory.
+ 2. Invoke one or multiple `checks` environments for each package identified as in scope.
+In the case of an environment invoking `pytest`, results can be collected in a junit xml file, and test markers can be selected via --mark_arg.
+"""
+ )
+
+ parser.add_argument(
+ "glob_string",
+ nargs="?",
+ help=(
+ "A comma separated list of glob strings that will target the top level directories that contain packages."
+ 'Examples: All = "azure-*", Single = "azure-keyvault-keys", Targeted Multiple = "azure-keyvault-keys,azure-mgmt-resource"'
+ ),
+ )
+
+ parser.add_argument(
+ "--junitxml",
+ dest="test_results",
+ help=(
+ "The output path for the test results file of invoked checks."
+ 'Example: --junitxml="junit/test-results.xml"'
+ ),
+ )
+
+ parser.add_argument(
+ "--mark_arg",
+ dest="mark_arg",
+ help=(
+ 'The complete argument for `pytest -m ""`. This can be used to exclude or include specific pytest markers.'
+ '--mark_arg="not cosmosEmulator"'
+ ),
+ )
+
+ parser.add_argument("--disablecov", help=("Flag. Disables code coverage."), action="store_true")
+
+ parser.add_argument(
+ "--service",
+ help=("Name of service directory (under sdk/) to test. Example: --service applicationinsights"),
+ )
+
+ parser.add_argument(
+ "-c",
+ "--checks",
+ dest="checks_list",
+ help="Specific set of named environments to execute",
+ )
+
+ parser.add_argument(
+ "-w",
+ "--wheel_dir",
+ dest="wheel_dir",
+ help="Location for prebuilt artifacts (if any)",
+ )
+
+ parser.add_argument(
+ "-i",
+ "--injected-packages",
+ dest="injected_packages",
+ default="",
+ help="Comma or space-separated list of packages that should be installed prior to dev_requirements. If local path, should be absolute.",
+ )
+
+ parser.add_argument(
+ "--filter-type",
+ dest="filter_type",
+ default="Build",
+ help="Filter type to identify eligible packages. for e.g. packages filtered in Build can pass filter type as Build,",
+ choices=["Build", "Docs", "Regression", "Omit_management", "None"],
+ )
+
+ parser.add_argument(
+ "-d",
+ "--dest-dir",
+ dest="dest_dir",
+ help="Location to generate any output files (if any). For e.g. APIView stub file",
+ )
+
+ parser.add_argument(
+ "--max-parallel",
+ dest="max_parallel",
+ type=int,
+ default=os.cpu_count() or 4,
+ help="Maximum number of concurrent checks (default: number of CPU cores).",
+ )
+
+ args = parser.parse_args()
+
+ configure_logging(args)
+
+ # We need to support both CI builds of everything and individual service
+ # folders. This logic allows us to do both.
+ if args.service and args.service != "auto":
+ service_dir = os.path.join("sdk", args.service)
+ target_dir = os.path.join(root_dir, service_dir)
+ else:
+ target_dir = root_dir
+
+ logger.info(f"Beginning discovery for {args.service} and root dir {root_dir}. Resolving to {target_dir}.")
+
+ if args.filter_type == "None":
+ args.filter_type = "Build"
+ compatibility_filter = False
+ else:
+ compatibility_filter = True
+
+ targeted_packages = discover_targeted_packages(
+ args.glob_string, target_dir, "", args.filter_type, compatibility_filter
+ )
+
+ if len(targeted_packages) == 0:
+ logger.info(f"No packages collected for targeting string {args.glob_string} and root dir {root_dir}. Exit 0.")
+ exit(0)
+
+ logger.info(f"Executing checks with the executable {sys.executable}.")
+ logger.info(f"Packages targeted: {targeted_packages}")
+
+ if args.wheel_dir:
+ os.environ["PREBUILT_WHEEL_DIR"] = args.wheel_dir
+
+ if not os.path.exists(os.path.join(root_dir, ".wheels")):
+ os.makedirs(os.path.join(root_dir, ".wheels"))
+
+ if in_ci():
+ # prepare a build of eng/tools/azure-sdk-tools
+ # todo: ensure that we honor this .wheels directory when replacing for dev reqs
+ build_whl_for_req("eng/tools/azure-sdk-tools", root_dir, os.path.join(root_dir, ".wheels"))
+
+ # so if we have checks whl,import_all and selected package paths `sdk/core/azure-core`, `sdk/storage/azure-storage-blob` we should
+ # shell out to `azypysdk ` with cwd of the package directory, which is what is in `targeted_packages` array
+ # each individual thread may need to re-invoke if they need to self-isolate themselves, but we don't have to worry about that.
+
+ # Prepare check list
+ raw_checks = (args.checks_list or "").split(",")
+ checks = [c.strip() for c in raw_checks if c and c.strip()]
+ if not checks:
+ logger.error("No valid checks provided via -c/--checks.")
+ sys.exit(2)
+
+ logger.info(f"Running {len(checks)} check(s) across {len(targeted_packages)} packages (max_parallel={args.max_parallel}).")
+
+ configure_interrupt_handling()
+ try:
+ exit_code = asyncio.run(run_all_checks(targeted_packages, checks, args.max_parallel))
+ except KeyboardInterrupt:
+ logger.error("Aborted by user.")
+ exit_code = 130
+ sys.exit(exit_code)
diff --git a/eng/tools/azure-sdk-tools/azpysdk/Check.py b/eng/tools/azure-sdk-tools/azpysdk/Check.py
index efdf05ec6ccf..cb3a3fa986c0 100644
--- a/eng/tools/azure-sdk-tools/azpysdk/Check.py
+++ b/eng/tools/azure-sdk-tools/azpysdk/Check.py
@@ -3,14 +3,15 @@
import argparse
import traceback
import sys
+import shutil
from typing import Sequence, Optional, List, Any, Tuple
from subprocess import check_call
+
from ci_tools.parsing import ParsedSetup
-from ci_tools.functions import discover_targeted_packages, get_venv_call
+from ci_tools.functions import discover_targeted_packages, get_venv_call, install_into_venv, get_venv_python
from ci_tools.variables import discover_repo_root
-from ci_tools.scenario import install_into_venv, get_venv_python
from ci_tools.logging import logger
# right now, we are assuming you HAVE to be in the azure-sdk-tools repo
@@ -18,6 +19,7 @@
# being called from within a site-packages folder. Due to that, we can't trust the location of __file__
REPO_ROOT = discover_repo_root()
+
class Check(abc.ABC):
"""
Base class for checks.
@@ -29,7 +31,9 @@ def __init__(self) -> None:
pass
@abc.abstractmethod
- def register(self, subparsers: "argparse._SubParsersAction", parent_parsers: Optional[List[argparse.ArgumentParser]] = None) -> None:
+ def register(
+ self, subparsers: "argparse._SubParsersAction", parent_parsers: Optional[List[argparse.ArgumentParser]] = None
+ ) -> None:
"""
Register this check with the CLI subparsers.
@@ -48,8 +52,15 @@ def run(self, args: argparse.Namespace) -> int:
def create_venv(self, isolate: bool, venv_location: str) -> str:
"""Abstraction for creating a virtual environment."""
- if (isolate):
+ if isolate:
venv_cmd = get_venv_call(sys.executable)
+ venv_python = get_venv_python(venv_location)
+ if os.path.exists(venv_python):
+ logger.info(f"Reusing existing venv at {venv_python}")
+ return venv_python
+ else:
+ shutil.rmtree(venv_location, ignore_errors=True)
+
check_call(venv_cmd + [venv_location])
# TODO: we should reuse part of build_whl_for_req to integrate with PREBUILT_WHL_DIR so that we don't have to fresh build for each
@@ -73,7 +84,6 @@ def get_executable(self, isolate: bool, check_name: str, executable: str, packag
os.makedirs(staging_directory, exist_ok=True)
return executable, staging_directory
-
def get_targeted_directories(self, args: argparse.Namespace) -> List[ParsedSetup]:
"""
Get the directories that are targeted for the check.
@@ -85,7 +95,9 @@ def get_targeted_directories(self, args: argparse.Namespace) -> List[ParsedSetup
try:
targeted.append(ParsedSetup.from_path(targeted_dir))
except Exception as e:
- logger.error("Error: Current directory does not appear to be a Python package (no setup.py or setup.cfg found). Remove '.' argument to run on child directories.")
+ logger.error(
+ "Error: Current directory does not appear to be a Python package (no setup.py or setup.cfg found). Remove '.' argument to run on child directories."
+ )
logger.error(f"Exception: {e}")
return []
else:
diff --git a/eng/tools/azure-sdk-tools/azpysdk/mypy.py b/eng/tools/azure-sdk-tools/azpysdk/mypy.py
index b074eadfbf50..27a7e0b819bf 100644
--- a/eng/tools/azure-sdk-tools/azpysdk/mypy.py
+++ b/eng/tools/azure-sdk-tools/azpysdk/mypy.py
@@ -8,34 +8,29 @@
from .Check import Check
from ci_tools.parsing import ParsedSetup
-from ci_tools.functions import pip_install
+from ci_tools.functions import install_into_venv
from ci_tools.scenario.generation import create_package_and_install
from ci_tools.variables import in_ci, set_envvar_defaults
-from ci_tools.environment_exclusions import (
- is_check_enabled, is_typing_ignored
-)
+from ci_tools.environment_exclusions import is_check_enabled, is_typing_ignored
from ci_tools.logging import logger
PYTHON_VERSION = "3.9"
MYPY_VERSION = "1.14.1"
+
class mypy(Check):
def __init__(self) -> None:
super().__init__()
- def register(self, subparsers: "argparse._SubParsersAction", parent_parsers: Optional[List[argparse.ArgumentParser]] = None) -> None:
- """Register the `mypy` check. The mypy check installs mypy and runs mypy against the target package.
- """
+ def register(
+ self, subparsers: "argparse._SubParsersAction", parent_parsers: Optional[List[argparse.ArgumentParser]] = None
+ ) -> None:
+ """Register the `mypy` check. The mypy check installs mypy and runs mypy against the target package."""
parents = parent_parsers or []
p = subparsers.add_parser("mypy", parents=parents, help="Run the mypy check")
p.set_defaults(func=self.run)
- p.add_argument(
- "--next",
- default=False,
- help="Next version of mypy is being tested",
- required=False
- )
+ p.add_argument("--next", default=False, help="Next version of mypy is being tested", required=False)
def run(self, args: argparse.Namespace) -> int:
"""Run the mypy check command."""
@@ -50,17 +45,17 @@ def run(self, args: argparse.Namespace) -> int:
for parsed in targeted:
package_dir = parsed.folder
package_name = parsed.name
-
+
executable, staging_directory = self.get_executable(args.isolate, args.command, sys.executable, package_dir)
logger.info(f"Processing {package_name} for mypy check")
# install mypy
try:
- if (args.next):
+ if args.next:
# use latest version of mypy
- pip_install(["mypy"], True, executable, package_dir)
+ install_into_venv(executable, "mypy", False)
else:
- pip_install([f"mypy=={MYPY_VERSION}"], True, executable, package_dir)
+ install_into_venv(executable, f"mypy=={MYPY_VERSION}", False)
except CalledProcessError as e:
logger.error("Failed to install mypy:", e)
return e.returncode
@@ -89,9 +84,7 @@ def run(self, args: argparse.Namespace) -> int:
src_code_error = None
sample_code_error = None
try:
- logger.info(
- f"Running mypy commands on src code: {src_code}"
- )
+ logger.info(f"Running mypy commands on src code: {src_code}")
results.append(check_call(src_code))
logger.info("Verified mypy, no issues found")
except CalledProcessError as src_error:
@@ -99,18 +92,14 @@ def run(self, args: argparse.Namespace) -> int:
results.append(src_error.returncode)
if not args.next and in_ci() and not is_check_enabled(package_dir, "type_check_samples", True):
- logger.info(
- f"Package {package_name} opts-out of mypy check on samples."
- )
+ logger.info(f"Package {package_name} opts-out of mypy check on samples.")
continue
else:
# check if sample dirs exists, if not, skip sample code check
samples = os.path.exists(os.path.join(package_dir, "samples"))
generated_samples = os.path.exists(os.path.join(package_dir, "generated_samples"))
if not samples and not generated_samples:
- logger.info(
- f"Package {package_name} does not have a samples directory."
- )
+ logger.info(f"Package {package_name} does not have a samples directory.")
else:
sample_code = [
*commands,
@@ -119,9 +108,7 @@ def run(self, args: argparse.Namespace) -> int:
os.path.join(package_dir, "samples" if samples else "generated_samples"),
]
try:
- logger.info(
- f"Running mypy commands on sample code: {sample_code}"
- )
+ logger.info(f"Running mypy commands on sample code: {sample_code}")
results.append(check_call(sample_code))
except CalledProcessError as sample_error:
sample_code_error = sample_error
@@ -129,6 +116,7 @@ def run(self, args: argparse.Namespace) -> int:
if args.next and in_ci() and not is_typing_ignored(package_name):
from gh_tools.vnext_issue_creator import create_vnext_issue, close_vnext_issue
+
if src_code_error or sample_code_error:
create_vnext_issue(package_dir, "mypy")
else:
diff --git a/eng/tools/azure-sdk-tools/ci_tools/functions.py b/eng/tools/azure-sdk-tools/ci_tools/functions.py
index 82c7e3b7cc28..de02a79bf865 100644
--- a/eng/tools/azure-sdk-tools/ci_tools/functions.py
+++ b/eng/tools/azure-sdk-tools/ci_tools/functions.py
@@ -179,8 +179,7 @@ def glob_packages(glob_string: str, target_root_dir: str) -> List[str]:
# drop any packages that exist within a tests or test directory
collected_top_level_directories = [
- p for p in collected_top_level_directories
- if not any(part in ("test", "tests") for part in p.split(os.sep))
+ p for p in collected_top_level_directories if not any(part in ("test", "tests") for part in p.split(os.sep))
]
# deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*"
@@ -411,7 +410,9 @@ def process_requires(setup_py_path: str, is_dev_build: bool = False):
else:
logging.info("Packages not available on PyPI:{}".format(requirement_to_update))
update_requires(setup_py_path, requirement_to_update)
- logging.info(f"Package requirement is updated in {'pyproject.toml' if pkg_details.is_pyproject else 'setup.py'}.")
+ logging.info(
+ f"Package requirement is updated in {'pyproject.toml' if pkg_details.is_pyproject else 'setup.py'}."
+ )
def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> Optional[str]:
@@ -444,7 +445,10 @@ def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> Optional[str]:
def pip_install(
- requirements: List[str], include_dependencies: bool = True, python_executable: Optional[str] = None, cwd: Optional[str] = None
+ requirements: List[str],
+ include_dependencies: bool = True,
+ python_executable: Optional[str] = None,
+ cwd: Optional[str] = None,
) -> bool:
"""
Attempts to invoke an install operation using the invoking python's pip. Empty requirements are auto-success.
@@ -474,6 +478,7 @@ def pip_uninstall(requirements: List[str], python_executable: str) -> bool:
"""
Attempts to invoke an install operation using the invoking python's pip. Empty requirements are auto-success.
"""
+ # we do not use get_pip_command here because uv pip doesn't have an uninstall command
exe = python_executable or sys.executable
command = [exe, "-m", "pip", "uninstall", "-y"]
@@ -489,6 +494,48 @@ def pip_uninstall(requirements: List[str], python_executable: str) -> bool:
return False
+def get_venv_python(venv_path: str) -> str:
+ """
+ Given a python venv path, identify the crossplat reference to the python executable.
+ """
+ # if we already have a path to a python executable, return it
+ if os.path.isfile(venv_path) and os.access(venv_path, os.X_OK) and os.path.basename(venv_path).startswith("python"):
+ return venv_path
+
+ # cross-platform python in a venv
+ bin_dir = "Scripts" if os.name == "nt" else "bin"
+ return os.path.join(venv_path, bin_dir, "python")
+
+
+def install_into_venv(
+ venv_path_or_executable: str, installation_target: str, editable: bool = True, extras: Optional[str] = None
+) -> None:
+ """
+ Install the package into an existing venv (venv_path) without activating it.
+
+ - Uses get_pip_command(get_venv_python) per request.
+ - If get_pip_command returns the 'uv' wrapper, we fall back to get_venv_python -m pip
+ so installation goes into the target venv reliably.
+ """
+ py = get_venv_python(venv_path_or_executable)
+ pip_cmd = get_pip_command(py)
+
+ install_target = installation_target
+ if extras:
+ install_target = f"{installation_target}[{extras}]"
+
+ if editable:
+ cmd = pip_cmd + ["install", "-e", install_target]
+ else:
+ cmd = pip_cmd + ["install", install_target]
+
+ if pip_cmd[0] == "uv":
+ cmd += ["--python", py]
+
+ # todo: clean this up so that we're using run_logged from #42862
+ subprocess.check_call(cmd)
+
+
def pip_install_requirements_file(requirements_file: str, python_executable: Optional[str] = None) -> bool:
return pip_install(["-r", requirements_file], True, python_executable)
@@ -906,7 +953,9 @@ def handle_incompatible_minimum_dev_reqs(
return cleansed_reqs
-def verify_package_classifiers(package_name: str, package_version: str, package_classifiers: List[str]) -> Tuple[bool, Optional[str]]:
+def verify_package_classifiers(
+ package_name: str, package_version: str, package_classifiers: List[str]
+) -> Tuple[bool, Optional[str]]:
"""
Verify that the package classifiers match the expected classifiers.
:param str package_name: The name of the package being verified. Used for detail in the error response.
@@ -924,7 +973,10 @@ def verify_package_classifiers(package_name: str, package_version: str, package_
if dev_status.is_prerelease:
for c in dev_classifiers:
if "4 - Beta" not in c:
- return False, f"{package_name} has version {package_version} and is a beta release, but has development status '{c}'. Expected 'Development Status :: 4 - Beta' ONLY."
+ return (
+ False,
+ f"{package_name} has version {package_version} and is a beta release, but has development status '{c}'. Expected 'Development Status :: 4 - Beta' ONLY.",
+ )
return True, None
# ga releases: all development statuses must be >= 5
@@ -935,9 +987,15 @@ def verify_package_classifiers(package_name: str, package_version: str, package_
# or Development Status :: 7 - Inactive
num = int(c.split("::")[1].split("-")[0].strip())
except (IndexError, ValueError):
- return False, f"{package_name} has version {package_version} and is a GA release, but failed to pull a status number from status '{c}'. Expecting format identical to 'Development Status :: 5 - Production/Stable'."
+ return (
+ False,
+ f"{package_name} has version {package_version} and is a GA release, but failed to pull a status number from status '{c}'. Expecting format identical to 'Development Status :: 5 - Production/Stable'.",
+ )
if num < 5:
- return False, f"{package_name} has version {package_version} and is a GA release, but had development status '{c}'. Expecting a development classifier that is equal or greater than 'Development Status :: 5 - Production/Stable'."
+ return (
+ False,
+ f"{package_name} has version {package_version} and is a GA release, but had development status '{c}'. Expecting a development classifier that is equal or greater than 'Development Status :: 5 - Production/Stable'.",
+ )
return True, None
@@ -951,14 +1009,15 @@ def get_venv_call(python_exe: Optional[str] = None) -> List[str]:
"""
# Check TOX_PIP_IMPL environment variable (aligns with tox.ini configuration)
- pip_impl = os.environ.get('TOX_PIP_IMPL', 'pip').lower()
+ pip_impl = os.environ.get("TOX_PIP_IMPL", "pip").lower()
# soon we will change this to default to uv
- if pip_impl == 'uv':
+ if pip_impl == "uv":
return ["uv", "venv"]
else:
return [python_exe if python_exe else sys.executable, "-m", "venv"]
+
def get_pip_command(python_exe: Optional[str] = None) -> List[str]:
"""
Determine whether to use 'uv pip' or regular 'pip' based on environment.
@@ -969,10 +1028,10 @@ def get_pip_command(python_exe: Optional[str] = None) -> List[str]:
"""
# Check TOX_PIP_IMPL environment variable (aligns with tox.ini configuration)
- pip_impl = os.environ.get('TOX_PIP_IMPL', 'pip').lower()
+ pip_impl = os.environ.get("TOX_PIP_IMPL", "pip").lower()
# soon we will change this to default to uv
- if pip_impl == 'uv':
+ if pip_impl == "uv":
return ["uv", "pip"]
else:
return [python_exe if python_exe else sys.executable, "-m", "pip"]
@@ -994,4 +1053,4 @@ def is_error_code_5_allowed(target_pkg: str, pkg_name: str):
):
return True
else:
- return False
\ No newline at end of file
+ return False
diff --git a/eng/tools/azure-sdk-tools/ci_tools/logging/__init__.py b/eng/tools/azure-sdk-tools/ci_tools/logging/__init__.py
index a32945c09978..2d0db7889aca 100644
--- a/eng/tools/azure-sdk-tools/ci_tools/logging/__init__.py
+++ b/eng/tools/azure-sdk-tools/ci_tools/logging/__init__.py
@@ -19,11 +19,11 @@ def configure_logging(
"""
# use cli arg > log level arg > env var
- if args.quiet:
+ if hasattr(args, "quiet") and args.quiet:
numeric_level = logging.ERROR
- elif args.verbose:
+ elif hasattr(args, "verbose") and args.verbose:
numeric_level = logging.DEBUG
- elif not args.log_level:
+ elif not getattr(args, "log_level", None):
numeric_level = getattr(logging, os.environ.get("LOGLEVEL", "INFO").upper())
else:
numeric_level = getattr(logging, args.log_level.upper(), None)
diff --git a/eng/tools/azure-sdk-tools/ci_tools/scenario/__init__.py b/eng/tools/azure-sdk-tools/ci_tools/scenario/__init__.py
index 2e079c44140b..3e634784c9d7 100644
--- a/eng/tools/azure-sdk-tools/ci_tools/scenario/__init__.py
+++ b/eng/tools/azure-sdk-tools/ci_tools/scenario/__init__.py
@@ -2,47 +2,8 @@
from subprocess import check_call
from .generation import prepare_and_test_optional
from .managed_virtual_env import ManagedVirtualEnv
-from ci_tools.functions import (
- get_pip_command
-)
from typing import Optional
# todo rename managed_virtual_env to virtual_env and move below functions there
-def get_venv_python(venv_path: str) -> str:
- """
- Given a python venv path, identify the crossplat reference to the python executable.
- """
- # cross-platform python in a venv
- bin_dir = "Scripts" if os.name == "nt" else "bin"
- return os.path.join(venv_path, bin_dir, "python")
-
-def install_into_venv(venv_path: str, package_path: str, editable: bool = True, extras: Optional[str] = None) -> None:
- """
- Install the package into an existing venv (venv_path) without activating it.
-
- - Uses get_pip_command(get_venv_python) per request.
- - If get_pip_command returns the 'uv' wrapper, we fall back to get_venv_python -m pip
- so installation goes into the target venv reliably.
- """
- py = get_venv_python(venv_path)
- pip_cmd = get_pip_command(py)
-
- install_target = package_path
- if extras:
- install_target = f"{package_path}[{extras}]"
-
- if editable:
- cmd = pip_cmd + ["install", "-e", install_target]
- else:
- cmd = pip_cmd + ["install", install_target]
-
- if pip_cmd[0] == "uv":
- cmd += ["--python", py]
-
- # Run the install; this will install into the interpreter referenced by `py` either by
- # the pip command or by calling uv with a target python env
- check_call(cmd)
-
-
-__all__ = ["prepare_and_test_optional", "ManagedVirtualEnv", "install_into_venv", "get_venv_python"]
+__all__ = ["prepare_and_test_optional", "ManagedVirtualEnv"]