From 9ffb371eb6ceb9b84a2b15dbf87ed13b8111cbd5 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 20 Jun 2025 19:16:00 +0000 Subject: [PATCH 01/13] remove all usage of deprecating pkg_resources --- eng/tox/prep_sphinx_env.py | 5 +---- eng/tox/run_sphinx_build.py | 5 ----- .../ci_tools/dependency_analysis.py | 13 ++++++------- tools/azure-sdk-tools/ci_tools/functions.py | 10 +++++----- .../ci_tools/parsing/parse_functions.py | 18 ++++++++---------- .../tests/test_conflict_resolution.py | 2 +- 6 files changed, 21 insertions(+), 32 deletions(-) diff --git a/eng/tox/prep_sphinx_env.py b/eng/tox/prep_sphinx_env.py index 1bab36a3b879..048497293ec3 100644 --- a/eng/tox/prep_sphinx_env.py +++ b/eng/tox/prep_sphinx_env.py @@ -12,11 +12,7 @@ import logging import shutil import argparse -from pkg_resources import Requirement -import ast import os -import textwrap -import io from tox_helper_tasks import ( unzip_sdist_to_directory, move_and_rename @@ -55,6 +51,7 @@ def should_build_docs(package_name): def create_index_file(readme_location, package_rst): readme_ext = os.path.splitext(readme_location)[1] + output = "" if readme_ext == ".md": with open(readme_location, "r") as file: output = file.read() diff --git a/eng/tox/run_sphinx_build.py b/eng/tox/run_sphinx_build.py index a4e389fd4198..de3e01b0d98c 100644 --- a/eng/tox/run_sphinx_build.py +++ b/eng/tox/run_sphinx_build.py @@ -12,14 +12,9 @@ import argparse import os import logging -import sys from prep_sphinx_env import should_build_docs from run_sphinx_apidoc import is_mgmt_package -from pkg_resources import Requirement -import ast import os -import textwrap -import io import shutil from ci_tools.parsing import ParsedSetup diff --git a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py index 8f268bdf5e70..680746596051 100755 --- a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py +++ b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import argparse -import ast from datetime import datetime import glob import io @@ -8,7 +7,6 @@ import os import re import sys -import textwrap from typing import List, Set, Dict, Tuple, Any try: @@ -16,8 +14,9 @@ except: from collections.abc import Sized -from pkg_resources import Requirement -from packaging.specifiers import SpecifierSet, Version +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.version import Version from ci_tools.variables import discover_repo_root from ci_tools.functions import discover_targeted_packages from ci_tools.parsing import ParsedSetup, parse_require @@ -30,7 +29,7 @@ pass # we only technically require this when outputting the rendered report -def get_known_versions(package_name: str) -> List[str]: +def get_known_versions(package_name: str) -> List[Version]: client = PyPIClient() return client.get_ordered_versions(package_name) @@ -60,8 +59,8 @@ def get_lib_deps(base_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Di packages = {} dependencies = {} for lib_dir in discover_targeted_packages("azure*", base_dir): + setup_path = os.path.join(lib_dir, "setup.py") try: - setup_path = os.path.join(lib_dir, "setup.py") parsed = ParsedSetup.from_path(setup_path) lib_name, version, requires = parsed.name, parsed.version, parsed.requires @@ -152,7 +151,7 @@ def dump_packages(data_pkgs: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, A def resolve_lib_deps(dump_data: Dict[str, Dict[str, Any]], data_pkgs: Dict[str, Dict[str, Any]], pkg_id: str) -> None: for dep in dump_data[pkg_id]["deps"]: - dep_req = Requirement.parse(dep["name"] + dep["version"]) + dep_req = Requirement(dep["name"] + dep["version"]) if dep["name"] in data_pkgs and data_pkgs[dep["name"]]["version"] in dep_req: # If the internal package version matches the dependency spec, # rewrite the dep version to match the internal package version diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 5274ff93b50d..6cb1066e6431 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -7,7 +7,7 @@ from ast import Not from packaging.specifiers import SpecifierSet from packaging.version import Version, parse, InvalidVersion -from pkg_resources import Requirement +from packaging.requirements import Requirement import io from ci_tools.variables import discover_repo_root, DEV_BUILD_IDENTIFIER, str_to_bool @@ -369,7 +369,7 @@ def process_requires(setup_py_path: str, is_dev_build: bool = False): """ pkg_details = ParsedSetup.from_path(setup_py_path) - azure_requirements = [Requirement.parse(r) for r in pkg_details.requires if r.startswith("azure")] + azure_requirements = [Requirement(r) for r in pkg_details.requires if r.startswith("azure")] # Find package requirements that are not available on PyPI requirement_to_update = {} @@ -673,7 +673,7 @@ def is_package_compatible( for immutable_requirement in immutable_requirements: for package_requirement in package_requirements: - if package_requirement.key == immutable_requirement.key: + if package_requirement.name == immutable_requirement.name: # if the dev_req line has a requirement that conflicts with the immutable requirement, # we need to resolve it. We KNOW that the immutable requirement will be of form package==version, # so we can reliably pull out the version and check it against the specifier of the dev_req line. @@ -743,7 +743,7 @@ def resolve_compatible_package(package_name: str, immutable_requirements: List[R """ pypi = PyPIClient() - immovable_pkgs = {req.key: req for req in immutable_requirements} + immovable_pkgs = {req.name: req for req in immutable_requirements} # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations # package as an example. @@ -935,7 +935,7 @@ def get_pip_command(python_exe: Optional[str] = None) -> List[str]: :param str python_exe: The Python executable to use (if not using the default). :return: List of command arguments for pip. :rtype: List[str] - + """ # Check TOX_PIP_IMPL environment variable (aligns with tox.ini configuration) pip_impl = os.environ.get('TOX_PIP_IMPL', 'pip').lower() diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index cea9c483f77b..85a27ab7218c 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -14,12 +14,8 @@ from typing import Dict, List, Tuple, Any, Optional -# Assumes the presence of setuptools -from pkg_resources import parse_requirements, Requirement - # this assumes the presence of "packaging" -from packaging.specifiers import SpecifierSet -import setuptools +from packaging.requirements import Requirement from setuptools import Extension from ci_tools.variables import str_to_bool @@ -313,7 +309,7 @@ def setup(*args, **kwargs): classifiers = kwargs.get("classifiers", []) keywords = kwargs.get("keywords", []) - is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x).key in NEW_REQ_PACKAGES), requires)) + is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x).name in NEW_REQ_PACKAGES), requires)) ext_package = kwargs.get("ext_package", None) ext_modules = kwargs.get("ext_modules", []) @@ -367,6 +363,8 @@ def parse_pyproject( project_config = toml_dict.get("project", None) + assert project_config is not None, f"Unable to find [project] section in {pyproject_filename}. Please ensure it is present." + # to pull a version from pyproject.toml, we need to get a dynamic version out. We can ask # setuptools to give us the metadata for a package, but that will involve _partially building_ the package # to create an egginfo folder. This is a very expensive operation goes against the entire point of @@ -393,7 +391,7 @@ def parse_pyproject( version = parsed_version python_requires = project_config.get("requires-python") requires = project_config.get("dependencies") - is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x).key in NEW_REQ_PACKAGES), requires)) + is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x).name in NEW_REQ_PACKAGES), requires)) name_space = name.replace("-", ".") package_data = get_value_from_dict(toml_dict, "tool.setuptools.package-data", None) include_package_data = get_value_from_dict(toml_dict, "tool.setuptools.include-package-data", True) @@ -568,11 +566,11 @@ def get_install_requires(setup_path: str) -> List[str]: def parse_require(req: str) -> Requirement: """ - Parses the incoming version specification and returns a tuple of the requirement name and specifier. + Parses a PEP 508 requirement string into a Requirement object. - "azure-core<2.0.0,>=1.11.0" -> [azure-core, <2.0.0,>=1.11.0] + Example: "azure-core<2.0.0,>=1.11.0" """ - return Requirement.parse(req) + return Requirement(req) def get_name_from_specifier(version: str) -> str: diff --git a/tools/azure-sdk-tools/tests/test_conflict_resolution.py b/tools/azure-sdk-tools/tests/test_conflict_resolution.py index 6baafe345d66..a758aebabe0b 100644 --- a/tools/azure-sdk-tools/tests/test_conflict_resolution.py +++ b/tools/azure-sdk-tools/tests/test_conflict_resolution.py @@ -5,7 +5,7 @@ from ci_tools.functions import resolve_compatible_package, is_package_compatible from typing import Optional, List from packaging.version import Version -from pkg_resources import Requirement +from packaging.requirements import Requirement @pytest.mark.parametrize( From f1cd1fa1929a702bc95dd9b6d085bf05f3b26743 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 20 Jun 2025 19:19:10 +0000 Subject: [PATCH 02/13] fix test --- tools/azure-sdk-tools/tests/test_parse_functionality.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/azure-sdk-tools/tests/test_parse_functionality.py b/tools/azure-sdk-tools/tests/test_parse_functionality.py index 35756f9a971f..05aa9814a4b3 100644 --- a/tools/azure-sdk-tools/tests/test_parse_functionality.py +++ b/tools/azure-sdk-tools/tests/test_parse_functionality.py @@ -31,10 +31,10 @@ def test_parse_require(): for scenario in test_scenarios: result = parse_require(scenario[0]) - assert result.key is not None + assert result.name is not None if scenario[2] is not None: assert len(result.specifier) != 0 - assert result.key == scenario[1] + assert result.name == scenario[1] assert str(result.specifier) == (scenario[2] or "") @@ -44,7 +44,7 @@ def test_parse_require_with_no_spec(): for scenario in spec_scenarios: result = parse_require(scenario) - assert result.key == scenario.replace("_", "-") + assert result.name == scenario.replace("_", "-") assert len(result.specifier) == 0 From cadbd0e91b9299724c9ed5b5f38deab6d9c25c2b Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 20 Jun 2025 19:24:58 +0000 Subject: [PATCH 03/13] fix issues with test_parse_functionality --- tools/azure-sdk-tools/tests/test_parse_functionality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/tests/test_parse_functionality.py b/tools/azure-sdk-tools/tests/test_parse_functionality.py index 05aa9814a4b3..49201b248930 100644 --- a/tools/azure-sdk-tools/tests/test_parse_functionality.py +++ b/tools/azure-sdk-tools/tests/test_parse_functionality.py @@ -17,7 +17,7 @@ def test_parse_require(): test_scenarios = [ - ("ConfigArgParse>=0.12.0", "configargparse", ">=0.12.0"), + ("ConfigArgParse>=0.12.0", "ConfigArgParse", ">=0.12.0"), ("msrest>=0.6.10", "msrest", ">=0.6.10"), ("azure-core<2.0.0,>=1.2.2", "azure-core", "<2.0.0,>=1.2.2"), ("msrest==0.6.10", "msrest", "==0.6.10"), @@ -44,7 +44,7 @@ def test_parse_require_with_no_spec(): for scenario in spec_scenarios: result = parse_require(scenario) - assert result.name == scenario.replace("_", "-") + assert result.name == scenario assert len(result.specifier) == 0 From 47d2bbbc9b31bed065ed7688b621fb238d3b8f60 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 20 Jun 2025 21:35:51 +0000 Subject: [PATCH 04/13] remove usage of Requirement.key in favor of Requirement.name --- tools/azure-sdk-tools/ci_tools/build.py | 2 +- tools/azure-sdk-tools/ci_tools/dependency_analysis.py | 4 ++-- tools/azure-sdk-tools/ci_tools/functions.py | 2 +- tools/azure-sdk-tools/ci_tools/scenario/generation.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index e4e17a5ad491..f564e95c723a 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -246,7 +246,7 @@ def create_package( # given the additional requirements of the package, we should install them in the current environment before attempting to build the package # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0` pip_output = get_pip_list_output(sys.executable) - necessary_install_requirements = [req for req in setup_parsed.requires if parse_require(req).key not in pip_output.keys()] + necessary_install_requirements = [req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys()] run([sys.executable, "-m", "pip", "install", *necessary_install_requirements], cwd=setup_parsed.folder) run([sys.executable, "-m", "build", f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", "-o", dist], cwd=setup_parsed.folder, check=True) else: diff --git a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py index 680746596051..2566b86fbc78 100755 --- a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py +++ b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py @@ -68,7 +68,7 @@ def get_lib_deps(base_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Di for req in requires: req_obj = parse_require(req) - req_name = req_obj.key + req_name = req_obj.name spec = req_obj.specifier if len(req_obj.specifier) else None if spec is None: spec = "" @@ -100,7 +100,7 @@ def get_wheel_deps(wheel_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, req = re.sub(r"[\s\(\)]", "", req) # Version specifiers appear in parentheses req_obj = parse_require(req) - req_name = req_obj.key + req_name = req_obj.name spec = req_obj.specifier if len(req_obj.specifier) else None if spec is None: spec = "" diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index dd05a5114188..e07c5b94cb0a 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -374,7 +374,7 @@ def process_requires(setup_py_path: str, is_dev_build: bool = False): # Find package requirements that are not available on PyPI requirement_to_update = {} for req in azure_requirements: - pkg_name = req.key + pkg_name = req.name spec = SpecifierSet(str(req).replace(pkg_name, "")) if not is_required_version_on_pypi(pkg_name, spec) or is_dev_build: diff --git a/tools/azure-sdk-tools/ci_tools/scenario/generation.py b/tools/azure-sdk-tools/ci_tools/scenario/generation.py index f6d9b05a5da7..7eb473e58d61 100644 --- a/tools/azure-sdk-tools/ci_tools/scenario/generation.py +++ b/tools/azure-sdk-tools/ci_tools/scenario/generation.py @@ -131,7 +131,7 @@ def create_package_and_install( # parse the specifier requirement = parse_require(req) - req_name = requirement.key + req_name = requirement.name req_specifier = requirement.specifier if len(requirement.specifier) else None # if we have the package already present... @@ -160,7 +160,7 @@ def create_package_and_install( ) except subprocess.CalledProcessError as e: requirement = parse_require(addition) - non_present_reqs.append(requirement.key) + non_present_reqs.append(requirement.name) additional_downloaded_reqs = [ os.path.abspath(os.path.join(tmp_dl_folder, pth)) for pth in os.listdir(tmp_dl_folder) From 1e1c817d02492d479f9233670bcaa18cc6fec79f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 20 Jun 2025 22:12:03 +0000 Subject: [PATCH 05/13] hopefully last fixes? --- scripts/devops_tasks/test_regression.py | 2 +- scripts/devops_tasks/update_regression_services.py | 3 +-- shared_requirements.txt | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/devops_tasks/test_regression.py b/scripts/devops_tasks/test_regression.py index 30498816113e..d9c2662ff55e 100644 --- a/scripts/devops_tasks/test_regression.py +++ b/scripts/devops_tasks/test_regression.py @@ -328,7 +328,7 @@ def find_package_dependency(glob_string, repo_root_dir, dependent_service): parsed = ParsedSetup.from_path(pkg_root) # Get a list of package names from install requires - required_pkgs = [parse_require(r).key for r in parsed.requires] + required_pkgs = [parse_require(r).name for r in parsed.requires] required_pkgs = [p for p in required_pkgs if p.startswith("azure")] for req_pkg in required_pkgs: diff --git a/scripts/devops_tasks/update_regression_services.py b/scripts/devops_tasks/update_regression_services.py index a76d862fe3c1..fa7155a1440a 100644 --- a/scripts/devops_tasks/update_regression_services.py +++ b/scripts/devops_tasks/update_regression_services.py @@ -9,7 +9,6 @@ import pdb import json -import pkg_resources from test_regression import find_package_dependency, AZURE_GLOB_STRING from ci_tools.functions import discover_targeted_packages @@ -85,7 +84,7 @@ def parse_service(pkg_path): print("The json file {} cannot be loaded.".format(args.json)) exit(1) - if len(service_list) > 0: + if len(service_list) > 0: settings = json.loads(settings_json) settings["matrix"]["DependentService"] = list(service_list) json_result = json.dumps(settings) diff --git a/shared_requirements.txt b/shared_requirements.txt index 3868d47ae0d2..ec36e25b7a8f 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -80,4 +80,4 @@ azure-monitor-opentelemetry python-dotenv pyrit prompty -jinja2 \ No newline at end of file +Jinja2 \ No newline at end of file From 6b4ff0d6417c7d09637870b8a5812cd81501caf9 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 20 Jun 2025 23:31:56 +0000 Subject: [PATCH 06/13] fix remaining issue with analyze_dependency as well --- eng/tox/install_depend_packages.py | 18 +++++++++--------- .../ci_tools/dependency_analysis.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index b5455371b2b9..92363c8d72c3 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -13,10 +13,10 @@ from subprocess import check_call from typing import TYPE_CHECKING, Callable, Optional -from pkg_resources import parse_version, Requirement from pypi_tools.pypi import PyPIClient from packaging.specifiers import SpecifierSet from packaging.version import Version +from packaging.requirements import Requirement from ci_tools.parsing import ParsedSetup, parse_require from ci_tools.functions import compare_python_version, handle_incompatible_minimum_dev_reqs, get_pip_command @@ -186,7 +186,7 @@ def process_bounded_versions(originating_pkg_name: str, pkg_name: str, versions: # lower bound general if pkg_name in MINIMUM_VERSION_GENERIC_OVERRIDES: versions = [ - v for v in versions if parse_version(v) >= parse_version(MINIMUM_VERSION_GENERIC_OVERRIDES[pkg_name]) + v for v in versions if Version(v) >= Version(MINIMUM_VERSION_GENERIC_OVERRIDES[pkg_name]) ] # lower bound platform-specific @@ -195,7 +195,7 @@ def process_bounded_versions(originating_pkg_name: str, pkg_name: str, versions: restrictions = PLATFORM_SPECIFIC_MINIMUM_OVERRIDES[platform_bound] if pkg_name in restrictions: - versions = [v for v in versions if parse_version(v) >= parse_version(restrictions[pkg_name])] + versions = [v for v in versions if Version(v) >= Version(restrictions[pkg_name])] # lower bound package-specific if ( @@ -205,13 +205,13 @@ def process_bounded_versions(originating_pkg_name: str, pkg_name: str, versions: versions = [ v for v in versions - if parse_version(v) >= parse_version(MINIMUM_VERSION_SPECIFIC_OVERRIDES[originating_pkg_name][pkg_name]) + if Version(v) >= Version(MINIMUM_VERSION_SPECIFIC_OVERRIDES[originating_pkg_name][pkg_name]) ] # upper bound general if pkg_name in MAXIMUM_VERSION_GENERIC_OVERRIDES: versions = [ - v for v in versions if parse_version(v) <= parse_version(MAXIMUM_VERSION_GENERIC_OVERRIDES[pkg_name]) + v for v in versions if Version(v) <= Version(MAXIMUM_VERSION_GENERIC_OVERRIDES[pkg_name]) ] # upper bound platform @@ -220,7 +220,7 @@ def process_bounded_versions(originating_pkg_name: str, pkg_name: str, versions: restrictions = PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES[platform_bound] if pkg_name in restrictions: - versions = [v for v in versions if parse_version(v) <= parse_version(restrictions[pkg_name])] + versions = [v for v in versions if Version(v) <= Version(restrictions[pkg_name])] # upper bound package-specific if ( @@ -230,7 +230,7 @@ def process_bounded_versions(originating_pkg_name: str, pkg_name: str, versions: versions = [ v for v in versions - if parse_version(v) <= parse_version(MAXIMUM_VERSION_SPECIFIC_OVERRIDES[originating_pkg_name][pkg_name]) + if Version(v) <= Version(MAXIMUM_VERSION_SPECIFIC_OVERRIDES[originating_pkg_name][pkg_name]) ] return versions @@ -241,7 +241,7 @@ def process_requirement(req, dependency_type, orig_pkg_name): # find package name and requirement specifier from requires requirement = parse_require(req) - pkg_name = requirement.key + pkg_name = requirement.name spec = requirement.specifier if len(requirement.specifier) else None # Filter out requirements with environment markers that don't match the current environment @@ -334,7 +334,7 @@ def filter_dev_requirements( # filter out any package available on PyPI (released_packages) # include packages without relative reference and packages not available on PyPI released_packages = [parse_require(p) for p in released_packages] - released_package_names = [p.key for p in released_packages] + released_package_names = [p.name for p in released_packages] # find prebuilt whl paths in dev requiremente prebuilt_dev_reqs = [os.path.basename(req.replace("\n", "")) for req in requirements if os.path.sep in req] # filter any req if wheel is for a released package diff --git a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py index 2566b86fbc78..59a5d15a9b45 100755 --- a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py +++ b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py @@ -152,7 +152,7 @@ def dump_packages(data_pkgs: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, A def resolve_lib_deps(dump_data: Dict[str, Dict[str, Any]], data_pkgs: Dict[str, Dict[str, Any]], pkg_id: str) -> None: for dep in dump_data[pkg_id]["deps"]: dep_req = Requirement(dep["name"] + dep["version"]) - if dep["name"] in data_pkgs and data_pkgs[dep["name"]]["version"] in dep_req: + if dep["name"] in data_pkgs and data_pkgs[dep["name"]]["version"] in dep_req.specifier: # If the internal package version matches the dependency spec, # rewrite the dep version to match the internal package version dep["version"] = data_pkgs[dep["name"]]["version"] From 8634ab5ae6d9e4baad9605a4ec1fced5588c48e7 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 25 Jun 2025 01:28:13 +0000 Subject: [PATCH 07/13] touch a few more files. dependencies.py and get_installed_pacackages within common_tasks.py will be harder --- eng/tox/verify_installed_packages.py | 4 ++-- scripts/devops_tasks/tox_harness.py | 6 +++--- scripts/multiapi_init_gen.py | 7 ++++--- sdk/core/azure-core/dev_requirements.txt | 1 + .../tests/async_tests/test_basic_transport_async.py | 6 +++--- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/eng/tox/verify_installed_packages.py b/eng/tox/verify_installed_packages.py index b39d93504146..afe6ee6c48c8 100644 --- a/eng/tox/verify_installed_packages.py +++ b/eng/tox/verify_installed_packages.py @@ -38,11 +38,11 @@ def verify_packages(package_file_path): for p in get_installed_packages(): if "==" in p: [package, version] = p.split("==") - installed[package.upper()] = version + installed[package.upper().replace("_","-")] = version expected = {} for p in packages: [package, version] = p.split("==") - expected[package.upper()] = version + expected[package.upper().replace("_","-")] = version missing_packages = [pkg for pkg in expected.keys() if installed.get(pkg) != expected.get(pkg)] diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index e7bf0b6e576b..9ffe33502591 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -21,7 +21,7 @@ from ci_tools.scenario.generation import replace_dev_reqs from ci_tools.functions import cleanup_directory from ci_tools.parsing import ParsedSetup -from pkg_resources import parse_requirements, RequirementParseError +from packaging.requirements import Requirement import logging logging.getLogger().setLevel(logging.INFO) @@ -58,7 +58,7 @@ def compare_req_to_injected_reqs(parsed_req, injected_packages): return any(parsed_req.name in req for req in injected_packages) - +# todo: verify this code def inject_custom_reqs(file, injected_packages, package_dir): req_lines = [] injected_packages = [p for p in re.split(r"[\s,]", injected_packages) if p] @@ -69,7 +69,7 @@ def inject_custom_reqs(file, injected_packages, package_dir): for line in f: logging.info("Attempting to parse {}".format(line)) try: - parsed_req = [req for req in parse_requirements(line)] + parsed_req = Requirement(line.strip()) except Exception as e: logging.error(e) parsed_req = [None] diff --git a/scripts/multiapi_init_gen.py b/scripts/multiapi_init_gen.py index dcb3e119ac4a..73eaffe6f722 100644 --- a/scripts/multiapi_init_gen.py +++ b/scripts/multiapi_init_gen.py @@ -40,9 +40,10 @@ ) import azure.common -import pkg_resources - -pkg_resources.declare_namespace("azure") +# all of the azure packages that are namespace packages have a __init__ that looks like: +# __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore +# so we can use the namespace package without explicitly declaring a parent azure namespace. At least according +# to the docs. _LOGGER = logging.getLogger(__name__) diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index 3e21785796c3..59b6fb6bd6d1 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -12,3 +12,4 @@ azure-data-tables opentelemetry-sdk~=1.26 opentelemetry-instrumentation-requests>=0.50b0 ../../identity/azure-identity +packaging # for version parsing in test_basic_transport_async.py \ No newline at end of file diff --git a/sdk/core/azure-core/tests/async_tests/test_basic_transport_async.py b/sdk/core/azure-core/tests/async_tests/test_basic_transport_async.py index 691996e7e71b..f0e11b1d213c 100644 --- a/sdk/core/azure-core/tests/async_tests/test_basic_transport_async.py +++ b/sdk/core/azure-core/tests/async_tests/test_basic_transport_async.py @@ -26,7 +26,7 @@ import sys import asyncio from unittest.mock import Mock -from pkg_resources import parse_version +from packaging.version import Version import aiohttp @@ -1050,7 +1050,7 @@ async def test_close_too_soon_works_fine(caplog, port, http_request): @pytest.mark.skipif( - parse_version(aiohttp.__version__) >= parse_version("3.10"), + Version(aiohttp.__version__) >= Version("3.10"), reason="aiohttp 3.10 introduced separate connection timeout", ) @pytest.mark.parametrize("http_request", HTTP_REQUESTS) @@ -1073,7 +1073,7 @@ async def test_aiohttp_timeout_response(http_request): @pytest.mark.skipif( - parse_version(aiohttp.__version__) < parse_version("3.10"), + Version(aiohttp.__version__) < Version("3.10"), reason="aiohttp 3.10 introduced separate connection timeout", ) @pytest.mark.parametrize("http_request", HTTP_REQUESTS) From 8b4f79ef60ec85213365a6fd640e8af740876c94 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 25 Jun 2025 02:25:50 +0000 Subject: [PATCH 08/13] progress. we need to have gotten rid of pkg_resources though so let's merge in the changes from remove-pkg-resources as well --- eng/tools/ci/azure-sdk-tools/README.md | 3 + eng/tools/ci/uv_checks/whl.py | 60 +++++++++++++++++++ scripts/devops_tasks/common_tasks.py | 20 ++++--- tools/azure-sdk-tools/ci_tools/uv/UVConfig.py | 10 ++++ 4 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 eng/tools/ci/azure-sdk-tools/README.md create mode 100644 eng/tools/ci/uv_checks/whl.py create mode 100644 tools/azure-sdk-tools/ci_tools/uv/UVConfig.py diff --git a/eng/tools/ci/azure-sdk-tools/README.md b/eng/tools/ci/azure-sdk-tools/README.md new file mode 100644 index 000000000000..5ad54468b27b --- /dev/null +++ b/eng/tools/ci/azure-sdk-tools/README.md @@ -0,0 +1,3 @@ +# Readme for azure-sdk-tools + +This will be the location of the moved azure-sdk-tools after it's moved from `tools/azure-sdk-tools` to here. diff --git a/eng/tools/ci/uv_checks/whl.py b/eng/tools/ci/uv_checks/whl.py new file mode 100644 index 000000000000..1e8f9e1e3cf9 --- /dev/null +++ b/eng/tools/ci/uv_checks/whl.py @@ -0,0 +1,60 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "wheel==0.45.1", +# "packaging==24.2", +# "urllib3==2.2.3", +# "tomli==2.2.1", +# "build==1.2.2.post1", +# "pytest", +# "pytest-cov", +# "azure-sdk-tools", +# "setuptools", +# ] +# +# [tool.uv.sources] +# azure-sdk-tools = { path = "../../../../tools/azure-sdk-tools", editable = true } +# /// + +import os +import argparse + +from ci_tools.functions import discover_targeted_packages + +from ci_tools.uv.UVConfig import UVConfig +from ci_tools.variables import discover_repo_root + +# — mimic tox’s setenv = +os.environ.setdefault("SPHINX_APIDOC_OPTIONS", "members,undoc-members,inherited-members") +os.environ.setdefault("PROXY_URL", "http://localhost:5000") +os.environ.setdefault("VIRTUALENV_WHEEL", "0.45.1") +os.environ.setdefault("VIRTUALENV_PIP", "24.0") +os.environ.setdefault("VIRTUALENV_SETUPTOOLS", "75.3.2") +os.environ.setdefault("PIP_EXTRA_INDEX_URL", "https://pypi.python.org/simple") + +def main(): + parser = argparse.ArgumentParser( + description="Run dev tooling against a given package directory" + ) + parser.add_argument( + "target", + nargs="?", + default="azure-core", + help="Path to the target package folder (default: current directory)", + ) + args = parser.parse_args() + + # todo: add some path params + print(args.target) + target_root_dir=discover_repo_root() + print(target_root_dir) + + breakpoint() + targeted = discover_targeted_packages(args.target, target_root_dir) + + breakpoint() + + +if __name__ == "__main__": + main() diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index 864bcaccc39f..5a2b97f5e82d 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -17,14 +17,12 @@ from subprocess import check_call, CalledProcessError, Popen from argparse import Namespace from typing import Iterable - -# Assumes the presence of setuptools -from pkg_resources import parse_version, parse_requirements, Requirement, WorkingSet, working_set +import importlib.metadata # this assumes the presence of "packaging" from packaging.specifiers import SpecifierSet -from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, NO_TESTS_ALLOWED, lambda_filter_azure_pkg, str_to_bool +from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, NO_TESTS_ALLOWED, lambda_filter_azure_pkg, get_pip_list_output from ci_tools.parsing import parse_require, ParsedSetup DEV_REQ_FILE = "dev_requirements.txt" @@ -247,9 +245,13 @@ def find_tools_packages(root_path): def get_installed_packages(paths=None): """Find packages in default or given lib paths""" - # WorkingSet returns installed packages in given path - # working_set returns installed packages in default path - # if paths is set then find installed packages from given paths - ws = WorkingSet(paths) if paths else working_set - return ["{0}=={1}".format(p.project_name, p.version) for p in ws] + # Use importlib.metadata.distributions, optionally searching provided paths + if paths: + dists = importlib.metadata.distributions(path=paths) + else: + dists = importlib.metadata.distributions() + + newWS = [f"{k}=={v}" for k, v in get_pip_list_output(sys.executable).items()] + + return newWS diff --git a/tools/azure-sdk-tools/ci_tools/uv/UVConfig.py b/tools/azure-sdk-tools/ci_tools/uv/UVConfig.py new file mode 100644 index 000000000000..39b930c77a62 --- /dev/null +++ b/tools/azure-sdk-tools/ci_tools/uv/UVConfig.py @@ -0,0 +1,10 @@ +import os + +class UVConfig: + """ + Static configuration for UV checks. + This class holds the static configuration values used in UV checks. + """ + + # The path to the UV checks configuration file + REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) From f685452fe7529f9afcf73ebe8931afaba645e4d5 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 25 Jun 2025 19:26:32 +0000 Subject: [PATCH 09/13] further updates --- eng/tools/ci/uv/README.md | 47 +++++++++++++++++++++++++++ eng/tools/ci/{uv_checks => uv}/whl.py | 16 +++++++-- 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 eng/tools/ci/uv/README.md rename eng/tools/ci/{uv_checks => uv}/whl.py (81%) diff --git a/eng/tools/ci/uv/README.md b/eng/tools/ci/uv/README.md new file mode 100644 index 000000000000..c06b0a91f5f8 --- /dev/null +++ b/eng/tools/ci/uv/README.md @@ -0,0 +1,47 @@ +# `uv` script checks for `azure-sdk-for-python` + +The scripts contained within this directory are self-contained validation scripts that are intended to be used by CI and for local development. + +## Example Check Invocations + +- `uv run tools/ci/uv/.py azure-core` +- `uv run tools/ci/uv/.py azure-storage*` +- `uv run tools/ci/uv/.py azure-storage-blob,azure-storage-queue` + +## Outstanding questions + +### Using tool instead of script? + +- Should we instead create a root `pyproject.toml` for each of these and install as a `uv tool`? + - ``` + tools/ + ci/ + uv/ + whl/ + whl.py + pyproject.toml + ``` + - Doing the above will allow us to install the check as a `tool` (which will have an isolated venv) and enable easy access via a named entrypoint on the PATH. + - We will need to be more explicit about cleaning up the venv being used in CI, versus `uv run` which is purely ephemeral unless told otherwise. + +### + +## Guidelines for creating a uv check + +- You **must** include `uv` in the list of dependencies for your script. This will enable access to `uv pip` from within the environment without assuming anything about a global `uv` install. This will enable `subprocess.run([sys.executable, '-m', 'uv', 'pip', 'install', 'packagename'])` with all the efficiency implied by using `uv`. +- You **must** `uv add --script tools/ci/uv/whl.py "-r eng/ci_tools.txt"` to ensure that the generic pinned dependencies are present on your script. +- You **must** cleave to the common argparse definition to ensure that all the checks have similar entrypoint behavior. + - This does not exist yet. + +## Transition from `tox` details + +### Pros + +- The speed is astonishing +- We get ephemeral venvs by default +- We can encode a ton more information due to the freedom of a pure-python script invoking all the work. + +### Cons + +- A single unified dependency file is not supported. When we update we will just have to `uv add --script -r eng/ci_tools.txt` which will update all the PEP723 preambles for our check scripts. +- We will need to parallelize invocation of these environments ourselves and deal with any gotchas. We'll do that in `dispatch_uv.py` which will be a follow-up to `dispatch_tox.py`. \ No newline at end of file diff --git a/eng/tools/ci/uv_checks/whl.py b/eng/tools/ci/uv/whl.py similarity index 81% rename from eng/tools/ci/uv_checks/whl.py rename to eng/tools/ci/uv/whl.py index 1e8f9e1e3cf9..a299d4bfe5be 100644 --- a/eng/tools/ci/uv_checks/whl.py +++ b/eng/tools/ci/uv/whl.py @@ -7,10 +7,22 @@ # "urllib3==2.2.3", # "tomli==2.2.1", # "build==1.2.2.post1", -# "pytest", -# "pytest-cov", +# "pytest==8.3.5", +# "pytest-cov==5.0.0", # "azure-sdk-tools", # "setuptools", +# "pytest-asyncio==0.24.0", +# "pytest-custom-exit-code==0.3.0", +# "pytest-xdist==3.2.1", +# "coverage==7.6.1", +# "bandit==1.6.2", +# "pyproject-api==1.8.0", +# "jinja2==3.1.6", +# "json-delta==2.0.2", +# "readme-renderer==43.0", +# "python-dotenv==1.0.1", +# "pyyaml==6.0.2", +# "six==1.17.0", # ] # # [tool.uv.sources] From 6f12aa4e26c22823085c1f69efc4542f85aaa5c6 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 25 Jun 2025 20:55:44 +0000 Subject: [PATCH 10/13] getting much closer --- eng/tools/ci/uv/README.md | 8 +++- eng/tools/ci/uv/whl.py | 48 ++++++++++++------- eng/tox/create_package_and_install.py | 2 +- tools/azure-sdk-tools/ci_tools/functions.py | 3 ++ tools/azure-sdk-tools/ci_tools/uv/UVConfig.py | 10 ---- tools/azure-sdk-tools/ci_tools/uv/__init__.py | 5 ++ tools/azure-sdk-tools/ci_tools/uv/invoke.py | 3 ++ tools/azure-sdk-tools/ci_tools/uv/manage.py | 25 ++++++++++ tools/azure-sdk-tools/ci_tools/uv/prepare.py | 25 ++++++++++ 9 files changed, 101 insertions(+), 28 deletions(-) delete mode 100644 tools/azure-sdk-tools/ci_tools/uv/UVConfig.py create mode 100644 tools/azure-sdk-tools/ci_tools/uv/__init__.py create mode 100644 tools/azure-sdk-tools/ci_tools/uv/invoke.py create mode 100644 tools/azure-sdk-tools/ci_tools/uv/manage.py create mode 100644 tools/azure-sdk-tools/ci_tools/uv/prepare.py diff --git a/eng/tools/ci/uv/README.md b/eng/tools/ci/uv/README.md index c06b0a91f5f8..641b70b7a4dd 100644 --- a/eng/tools/ci/uv/README.md +++ b/eng/tools/ci/uv/README.md @@ -24,7 +24,13 @@ The scripts contained within this directory are self-contained validation script - Doing the above will allow us to install the check as a `tool` (which will have an isolated venv) and enable easy access via a named entrypoint on the PATH. - We will need to be more explicit about cleaning up the venv being used in CI, versus `uv run` which is purely ephemeral unless told otherwise. -### +### How do I debug when I'm modifying a new `uv` check? + +todo: once scbedd r someone else has a good story for this in vscode + pylance. + +### Is there a good way to pass on custom arguments that are after the -- in the uv script invocation + +todo: discover this ## Guidelines for creating a uv check diff --git a/eng/tools/ci/uv/whl.py b/eng/tools/ci/uv/whl.py index a299d4bfe5be..2622dee8a4f7 100644 --- a/eng/tools/ci/uv/whl.py +++ b/eng/tools/ci/uv/whl.py @@ -5,7 +5,7 @@ # "wheel==0.45.1", # "packaging==24.2", # "urllib3==2.2.3", -# "tomli==2.2.1", +# "tomli", # "build==1.2.2.post1", # "pytest==8.3.5", # "pytest-cov==5.0.0", @@ -23,6 +23,7 @@ # "python-dotenv==1.0.1", # "pyyaml==6.0.2", # "six==1.17.0", +# "uv", # ] # # [tool.uv.sources] @@ -34,16 +35,12 @@ from ci_tools.functions import discover_targeted_packages -from ci_tools.uv.UVConfig import UVConfig +from ci_tools.uv import install_uv_devrequirements, set_environment_variable_defaults, DEFAULT_ENVIRONMENT_VARIABLES, uv_pytest from ci_tools.variables import discover_repo_root - -# — mimic tox’s setenv = -os.environ.setdefault("SPHINX_APIDOC_OPTIONS", "members,undoc-members,inherited-members") -os.environ.setdefault("PROXY_URL", "http://localhost:5000") -os.environ.setdefault("VIRTUALENV_WHEEL", "0.45.1") -os.environ.setdefault("VIRTUALENV_PIP", "24.0") -os.environ.setdefault("VIRTUALENV_SETUPTOOLS", "75.3.2") -os.environ.setdefault("PIP_EXTRA_INDEX_URL", "https://pypi.python.org/simple") +from ci_tools.scenario.generation import create_package_and_install +import tempfile +import shutil +import os def main(): parser = argparse.ArgumentParser( @@ -57,16 +54,35 @@ def main(): ) args = parser.parse_args() - # todo: add some path params - print(args.target) - target_root_dir=discover_repo_root() - print(target_root_dir) + set_environment_variable_defaults(DEFAULT_ENVIRONMENT_VARIABLES) - breakpoint() + # todo, we should use os.cwd as the target if no target is provided + # with some validation to ensure we're in a package directory (eg setup.py or pyproject.toml exists) and not repo root. + target_root_dir=discover_repo_root() targeted = discover_targeted_packages(args.target, target_root_dir) - breakpoint() + failed = False + + for pkg in targeted: + install_uv_devrequirements( + pkg_path=pkg, + allow_nonpresence=True, + ) + + staging_area = tempfile.mkdtemp() + + create_package_and_install( + distribution_directory=staging_area, + target_setup=pkg, + skip_install=False, + cache_dir=None, + work_dir=staging_area, + force_create=False, + package_type="wheel", + pre_download_disabled=False, + ) + uv_pytest(target_path=pkg, additional_args=["--cov", "--cov-report=xml", "--cov-report=html"]) if __name__ == "__main__": main() diff --git a/eng/tox/create_package_and_install.py b/eng/tox/create_package_and_install.py index 15c5bf926f9b..6d8f6317ca6e 100644 --- a/eng/tox/create_package_and_install.py +++ b/eng/tox/create_package_and_install.py @@ -94,4 +94,4 @@ ) - + diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 08cef76625c4..b08a44acc7d7 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -943,5 +943,8 @@ def get_pip_command(python_exe: Optional[str] = None) -> List[str]: if pip_impl == 'uv': return ["uv", "pip"] + # using environment uv and not depending on global uv install + elif os.environ.get('IN_UV', None) == '1': + return [python_exe if python_exe else sys.executable, "-m", "uv", "pip"] else: return [python_exe if python_exe else sys.executable, "-m", "pip"] diff --git a/tools/azure-sdk-tools/ci_tools/uv/UVConfig.py b/tools/azure-sdk-tools/ci_tools/uv/UVConfig.py deleted file mode 100644 index 39b930c77a62..000000000000 --- a/tools/azure-sdk-tools/ci_tools/uv/UVConfig.py +++ /dev/null @@ -1,10 +0,0 @@ -import os - -class UVConfig: - """ - Static configuration for UV checks. - This class holds the static configuration values used in UV checks. - """ - - # The path to the UV checks configuration file - REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) diff --git a/tools/azure-sdk-tools/ci_tools/uv/__init__.py b/tools/azure-sdk-tools/ci_tools/uv/__init__.py new file mode 100644 index 000000000000..ddcace71f26f --- /dev/null +++ b/tools/azure-sdk-tools/ci_tools/uv/__init__.py @@ -0,0 +1,5 @@ +from .manage import install_uv_devrequirements +from .prepare import set_environment_variable_defaults, DEFAULT_ENVIRONMENT_VARIABLES +from .invoke import uv_pytest + +__all__ = ["install_uv_devrequirements", "set_environment_variable_defaults", "DEFAULT_ENVIRONMENT_VARIABLES", "uv_pytest"] \ No newline at end of file diff --git a/tools/azure-sdk-tools/ci_tools/uv/invoke.py b/tools/azure-sdk-tools/ci_tools/uv/invoke.py new file mode 100644 index 000000000000..5abaadee494d --- /dev/null +++ b/tools/azure-sdk-tools/ci_tools/uv/invoke.py @@ -0,0 +1,3 @@ +def uv_pytest(target_path: str, additional_args: list[str] = []) -> bool: + print(f"Invoke pytest for {target_path}") + return False \ No newline at end of file diff --git a/tools/azure-sdk-tools/ci_tools/uv/manage.py b/tools/azure-sdk-tools/ci_tools/uv/manage.py new file mode 100644 index 000000000000..76b44551f598 --- /dev/null +++ b/tools/azure-sdk-tools/ci_tools/uv/manage.py @@ -0,0 +1,25 @@ +import os +import logging +import subprocess +import sys + +def install_uv_devrequirements(pkg_path: str, allow_nonpresence: bool = False): + """ + Installs the development requirements for a given package. + + Args: + pkg_path (str): The path to the package directory. + """ + + dev_req_path = os.path.join(pkg_path, "dev_requirements.txt") + + if not os.path.exists(dev_req_path) and not allow_nonpresence: + print(f"Development requirements file not found at {dev_req_path}") + raise FileNotFoundError(f"Development requirements file not found at {dev_req_path}") + + try: + command = [sys.executable, "-m", "uv", "pip", "install", "-r", dev_req_path] + subprocess.run(command, check=True, cwd=pkg_path) + logging.info(f"Successfully installed development requirements from {dev_req_path}.") + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to install development requirements: {e}") diff --git a/tools/azure-sdk-tools/ci_tools/uv/prepare.py b/tools/azure-sdk-tools/ci_tools/uv/prepare.py new file mode 100644 index 000000000000..e7a2dfa2276c --- /dev/null +++ b/tools/azure-sdk-tools/ci_tools/uv/prepare.py @@ -0,0 +1,25 @@ +from typing import Dict +import os + +DEFAULT_ENVIRONMENT_VARIABLES = { + "SPHINX_APIDOC_OPTIONS": "members,undoc-members,inherited-members", + "PROXY_URL": "http://localhost:5000", + "VIRTUALENV_WHEEL": "0.45.1", + "VIRTUALENV_PIP": "24.0", + "VIRTUALENV_SETUPTOOLS": "75.3.2", + "PIP_EXTRA_INDEX_URL": "https://pypi.python.org/simple", + # I haven't spent much time looking to see if a variable exists when invoking uv run. there might be one already that we can depend + # on for get_pip_command adjustment. + "IN_UV": "1" +} + +def set_environment_variable_defaults(settings: Dict[str, str]) -> None: + """ + Sets default environment variables for the UV prepare script. + + Args: + settings (Dict[str, str]): A dictionary of environment variable names and their default values. + """ + for key, value in settings.items(): + if key not in os.environ: + os.environ.setdefault(key, value) \ No newline at end of file From 465e3b54e05828632ae96e6c94af56a49ce7f2d5 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 25 Jun 2025 21:00:52 +0000 Subject: [PATCH 11/13] save additional readme update --- eng/tools/ci/uv/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eng/tools/ci/uv/README.md b/eng/tools/ci/uv/README.md index 641b70b7a4dd..f944509ae7c7 100644 --- a/eng/tools/ci/uv/README.md +++ b/eng/tools/ci/uv/README.md @@ -23,6 +23,15 @@ The scripts contained within this directory are self-contained validation script ``` - Doing the above will allow us to install the check as a `tool` (which will have an isolated venv) and enable easy access via a named entrypoint on the PATH. - We will need to be more explicit about cleaning up the venv being used in CI, versus `uv run` which is purely ephemeral unless told otherwise. +- Another option would be to: + - ``` + tools/ + ci/ + uv/ + whl.py + mindependency.py + pyproject.toml -> reference both + ``` ### How do I debug when I'm modifying a new `uv` check? From a07f6872cefbe419c4103bef737aec31cb0d297e Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 25 Jun 2025 21:04:12 +0000 Subject: [PATCH 12/13] rename whl to azpy-whl --- eng/tools/ci/uv/README.md | 1 + eng/tools/ci/uv/{whl.py => azpy-whl.py} | 0 2 files changed, 1 insertion(+) rename eng/tools/ci/uv/{whl.py => azpy-whl.py} (100%) diff --git a/eng/tools/ci/uv/README.md b/eng/tools/ci/uv/README.md index f944509ae7c7..d643ca4b3222 100644 --- a/eng/tools/ci/uv/README.md +++ b/eng/tools/ci/uv/README.md @@ -45,6 +45,7 @@ todo: discover this - You **must** include `uv` in the list of dependencies for your script. This will enable access to `uv pip` from within the environment without assuming anything about a global `uv` install. This will enable `subprocess.run([sys.executable, '-m', 'uv', 'pip', 'install', 'packagename'])` with all the efficiency implied by using `uv`. - You **must** `uv add --script tools/ci/uv/whl.py "-r eng/ci_tools.txt"` to ensure that the generic pinned dependencies are present on your script. +- You **must** preface your scripts with `azpy`. So the `whl` check would be `azpy-whl.py`. - You **must** cleave to the common argparse definition to ensure that all the checks have similar entrypoint behavior. - This does not exist yet. diff --git a/eng/tools/ci/uv/whl.py b/eng/tools/ci/uv/azpy-whl.py similarity index 100% rename from eng/tools/ci/uv/whl.py rename to eng/tools/ci/uv/azpy-whl.py From 23348878e94d997d5ff4a784c74d417cf9e32ddd Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Thu, 26 Jun 2025 13:09:54 -0700 Subject: [PATCH 13/13] fix the download even if pip download is slow in comparison uv (#41801) Co-authored-by: Scott Beddall --- eng/tools/ci/uv/azpy-whl.py | 3 +- scripts/devops_tasks/common_tasks.py | 19 +--------- scripts/devops_tasks/tox_harness.py | 3 +- tools/azure-sdk-tools/ci_tools/functions.py | 18 ++++++++++ .../ci_tools/scenario/generation.py | 6 +++- tools/azure-sdk-tools/ci_tools/uv/invoke.py | 35 +++++++++++++++++-- 6 files changed, 59 insertions(+), 25 deletions(-) diff --git a/eng/tools/ci/uv/azpy-whl.py b/eng/tools/ci/uv/azpy-whl.py index 2622dee8a4f7..06b85367556c 100644 --- a/eng/tools/ci/uv/azpy-whl.py +++ b/eng/tools/ci/uv/azpy-whl.py @@ -82,7 +82,8 @@ def main(): pre_download_disabled=False, ) - uv_pytest(target_path=pkg, additional_args=["--cov", "--cov-report=xml", "--cov-report=html"]) + # todo, come up with a good pattern for passing all the additional args after -- to pytest + result = uv_pytest(target_path=pkg, additional_args=["--cov", "--cov-report=xml", "--cov-report=html"]) if __name__ == "__main__": main() diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index 5a2b97f5e82d..9f4a79001821 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -22,7 +22,7 @@ # this assumes the presence of "packaging" from packaging.specifiers import SpecifierSet -from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, NO_TESTS_ALLOWED, lambda_filter_azure_pkg, get_pip_list_output +from ci_tools.functions import lambda_filter_azure_pkg, get_pip_list_output from ci_tools.parsing import parse_require, ParsedSetup DEV_REQ_FILE = "dev_requirements.txt" @@ -112,23 +112,6 @@ def create_code_coverage_params(parsed_args: Namespace, package_path: str): return coverage_args -# This function returns if error code 5 is allowed for a given package -def is_error_code_5_allowed(target_pkg, pkg_name): - if ( - all( - map( - lambda x: any([pkg_id in x for pkg_id in MANAGEMENT_PACKAGE_IDENTIFIERS]), - [target_pkg], - ) - ) - or pkg_name in MANAGEMENT_PACKAGE_IDENTIFIERS - or pkg_name in NO_TESTS_ALLOWED - ): - return True - else: - return False - - # This method installs package from a pre-built whl def install_package_from_whl(package_whl_path, working_dir, python_sym_link=sys.executable): commands = [ diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index 9ffe33502591..21ddd707ab79 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -11,7 +11,6 @@ from common_tasks import ( run_check_call, clean_coverage, - is_error_code_5_allowed, create_code_coverage_params, ) @@ -19,7 +18,7 @@ from ci_tools.environment_exclusions import filter_tox_environment_string from ci_tools.ci_interactions import output_ci_warning from ci_tools.scenario.generation import replace_dev_reqs -from ci_tools.functions import cleanup_directory +from ci_tools.functions import cleanup_directory, is_error_code_5_allowed from ci_tools.parsing import ParsedSetup from packaging.requirements import Requirement import logging diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index b08a44acc7d7..eeeedc56a088 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -948,3 +948,21 @@ def get_pip_command(python_exe: Optional[str] = None) -> List[str]: return [python_exe if python_exe else sys.executable, "-m", "uv", "pip"] else: return [python_exe if python_exe else sys.executable, "-m", "pip"] + +def is_error_code_5_allowed(target_pkg: str, pkg_name: str): + """ + Determine if error code 5 (no pytests run) is allowed for the given package. + """ + if ( + all( + map( + lambda x: any([pkg_id in x for pkg_id in MANAGEMENT_PACKAGE_IDENTIFIERS]), + [target_pkg], + ) + ) + or pkg_name in MANAGEMENT_PACKAGE_IDENTIFIERS + or pkg_name in NO_TESTS_ALLOWED + ): + return True + else: + return False \ No newline at end of file diff --git a/tools/azure-sdk-tools/ci_tools/scenario/generation.py b/tools/azure-sdk-tools/ci_tools/scenario/generation.py index 7eb473e58d61..d96de04ee058 100644 --- a/tools/azure-sdk-tools/ci_tools/scenario/generation.py +++ b/tools/azure-sdk-tools/ci_tools/scenario/generation.py @@ -112,7 +112,11 @@ def create_package_and_install( ) pip_cmd = get_pip_command(python_exe) - download_command = pip_cmd + [ + + download_command = [ + sys.executable, + "-m", + "pip", # uv pip doesn't have a download command, so we use the system pip "download", "-d", tmp_dl_folder, diff --git a/tools/azure-sdk-tools/ci_tools/uv/invoke.py b/tools/azure-sdk-tools/ci_tools/uv/invoke.py index 5abaadee494d..be45fb98753e 100644 --- a/tools/azure-sdk-tools/ci_tools/uv/invoke.py +++ b/tools/azure-sdk-tools/ci_tools/uv/invoke.py @@ -1,3 +1,32 @@ -def uv_pytest(target_path: str, additional_args: list[str] = []) -> bool: - print(f"Invoke pytest for {target_path}") - return False \ No newline at end of file +import subprocess +import logging + +from pytest import main as pytest_main + +from ci_tools.functions import is_error_code_5_allowed + +def uv_pytest(target_path: str, additional_args: list[str] = [], shell: bool = False) -> bool: + logging.info(f"Invoke pytest for {target_path}") + + exit_code = 0 + + if shell: + result = subprocess.run( + [sys.executable, "-m", "pytest", target_path] + additional_args, + check=False + ) + exit_code = result.returncode + else: + exit_code = pytest_main( + [target_path] + additional_args + ) + + if exit_code != 0: + if exit_code == 5 and is_error_code_5_allowed(): + logging.info("Exit code 5 is allowed, continuing execution.") + return True + else: + logging.info(f"pytest failed with exit code {exit_code}.") + return False + + return True \ No newline at end of file