From f5f6945272f1be8dfe9fc3be1bc6ce3b4d80bdad Mon Sep 17 00:00:00 2001 From: mccoyp Date: Wed, 16 Jul 2025 00:58:08 +0000 Subject: [PATCH 1/5] Check for pyproject.toml --- .../ci_tools/dependency_analysis.py | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py index 59a5d15a9b45..6f6444656855 100755 --- a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py +++ b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py @@ -58,25 +58,34 @@ def record_dep(dependencies: Dict[str, Dict[str, Any]], req_name: str, spec: str def get_lib_deps(base_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]: packages = {} dependencies = {} - for lib_dir in discover_targeted_packages("azure*", base_dir): - setup_path = os.path.join(lib_dir, "setup.py") - try: - parsed = ParsedSetup.from_path(setup_path) - lib_name, version, requires = parsed.name, parsed.version, parsed.requires - packages[lib_name] = {"version": version, "source": lib_dir, "deps": []} + def parse_setup(setup_path: str) -> None: + parsed = ParsedSetup.from_path(setup_path) + lib_name, version, requires = parsed.name, parsed.version, parsed.requires + + packages[lib_name] = {"version": version, "source": lib_dir, "deps": []} + + for req in requires: + req_obj = parse_require(req) + req_name = req_obj.name + spec = req_obj.specifier if len(req_obj.specifier) else None + if spec is None: + spec = "" - for req in requires: - req_obj = parse_require(req) - req_name = req_obj.name - spec = req_obj.specifier if len(req_obj.specifier) else None - if spec is None: - spec = "" + packages[lib_name]["deps"].append({"name": req_name, "version": str(spec)}) + record_dep(dependencies, req_name, str(spec), lib_name) - packages[lib_name]["deps"].append({"name": req_name, "version": str(spec)}) - record_dep(dependencies, req_name, str(spec), lib_name) + for lib_dir in discover_targeted_packages("azure*", base_dir): + setup_path = os.path.join(lib_dir, "setup.py") + try: + parse_setup(setup_path) + except: + pass + pyproject_path = os.path.join(lib_dir, "pyproject.toml") + try: + parse_setup(pyproject_path) except: - print("Failed to parse %s" % (setup_path)) + print(f"Failed to parse {setup_path} or {pyproject_path}") return packages, dependencies From 0c4d11f3a309c317225f4587330c0f69a40c9fb2 Mon Sep 17 00:00:00 2001 From: mccoyp Date: Wed, 16 Jul 2025 01:19:43 +0000 Subject: [PATCH 2/5] Only attempt pyproject parsing if setup.py fails --- tools/azure-sdk-tools/ci_tools/dependency_analysis.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py index 6f6444656855..f06a0c961f99 100755 --- a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py +++ b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py @@ -80,12 +80,11 @@ def parse_setup(setup_path: str) -> None: try: parse_setup(setup_path) except: - pass - pyproject_path = os.path.join(lib_dir, "pyproject.toml") - try: - parse_setup(pyproject_path) - except: - print(f"Failed to parse {setup_path} or {pyproject_path}") + pyproject_path = os.path.join(lib_dir, "pyproject.toml") + try: + parse_setup(pyproject_path) + except: + print(f"Failed to parse {setup_path} or {pyproject_path}") return packages, dependencies From c111e863840e6adda9f83bdaa6c0ed0d35700b66 Mon Sep 17 00:00:00 2001 From: mccoyp Date: Thu, 17 Jul 2025 01:30:28 +0000 Subject: [PATCH 3/5] Add test and documentation --- .../azure-sdk-tools/ci_tools/dependency_analysis.py | 3 +++ .../tests/test_pyproject_interactions.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py index f06a0c961f99..c5d6fb270048 100755 --- a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py +++ b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py @@ -60,6 +60,7 @@ def get_lib_deps(base_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Di dependencies = {} def parse_setup(setup_path: str) -> None: + """Attempts to parse either a setup.py or pyproject.toml file to extract package information.""" parsed = ParsedSetup.from_path(setup_path) lib_name, version, requires = parsed.name, parsed.version, parsed.requires @@ -79,10 +80,12 @@ def parse_setup(setup_path: str) -> None: setup_path = os.path.join(lib_dir, "setup.py") try: parse_setup(setup_path) + # If we can't parse setup.py, we try to parse pyproject.toml except: pyproject_path = os.path.join(lib_dir, "pyproject.toml") try: parse_setup(pyproject_path) + # If we can't parse pyproject.toml either, we print an error message with attempted paths except: print(f"Failed to parse {setup_path} or {pyproject_path}") return packages, dependencies diff --git a/tools/azure-sdk-tools/tests/test_pyproject_interactions.py b/tools/azure-sdk-tools/tests/test_pyproject_interactions.py index e5a10faded72..afcbf180f8a2 100644 --- a/tools/azure-sdk-tools/tests/test_pyproject_interactions.py +++ b/tools/azure-sdk-tools/tests/test_pyproject_interactions.py @@ -3,8 +3,10 @@ import pytest +from ci_tools.dependency_analysis import get_lib_deps from ci_tools.parsing import update_build_config, get_build_config, get_config_setting from ci_tools.environment_exclusions import is_check_enabled +from ci_tools.variables import discover_repo_root integration_folder = os.path.join(os.path.dirname(__file__), "integration") pyproject_folder = os.path.join(integration_folder, "scenarios", "pyproject_build_config") @@ -88,4 +90,13 @@ def test_pyproject_update_check_override(): assert update_result == build_config reloaded_build_config = get_build_config(temp_dir) - assert reloaded_build_config == update_result \ No newline at end of file + assert reloaded_build_config == update_result + + +def test_pyproject_get_lib_deps(): + all_packages, _ = get_lib_deps(discover_repo_root()) + # Ensure that libraries with pyproject.toml files are fetched correctly; azure-keyvault-keys is an example + pyproject_info = all_packages["azure-keyvault-keys"] + assert pyproject_info["version"] + assert pyproject_info["source"] + assert len(pyproject_info["deps"]) > 0 From 8d28d7be4e73f85c61f86d137aad804b3881425d Mon Sep 17 00:00:00 2001 From: mccoyp Date: Thu, 17 Jul 2025 22:08:29 +0000 Subject: [PATCH 4/5] Pass lib_dir to from_path instead of file path --- .../ci_tools/dependency_analysis.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py index c5d6fb270048..54b61d3a1fc6 100755 --- a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py +++ b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py @@ -59,9 +59,8 @@ def get_lib_deps(base_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Di packages = {} dependencies = {} - def parse_setup(setup_path: str) -> None: - """Attempts to parse either a setup.py or pyproject.toml file to extract package information.""" - parsed = ParsedSetup.from_path(setup_path) + for lib_dir in discover_targeted_packages("azure*", base_dir): + parsed = ParsedSetup.from_path(lib_dir) lib_name, version, requires = parsed.name, parsed.version, parsed.requires packages[lib_name] = {"version": version, "source": lib_dir, "deps": []} @@ -75,19 +74,6 @@ def parse_setup(setup_path: str) -> None: packages[lib_name]["deps"].append({"name": req_name, "version": str(spec)}) record_dep(dependencies, req_name, str(spec), lib_name) - - for lib_dir in discover_targeted_packages("azure*", base_dir): - setup_path = os.path.join(lib_dir, "setup.py") - try: - parse_setup(setup_path) - # If we can't parse setup.py, we try to parse pyproject.toml - except: - pyproject_path = os.path.join(lib_dir, "pyproject.toml") - try: - parse_setup(pyproject_path) - # If we can't parse pyproject.toml either, we print an error message with attempted paths - except: - print(f"Failed to parse {setup_path} or {pyproject_path}") return packages, dependencies From 0fe35e3da5dd03d451489c0904cd5b5ee6df5773 Mon Sep 17 00:00:00 2001 From: mccoyp Date: Thu, 17 Jul 2025 22:10:58 +0000 Subject: [PATCH 5/5] Updated parsing exception --- .../ci_tools/dependency_analysis.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py index 54b61d3a1fc6..d4192ff858a5 100755 --- a/tools/azure-sdk-tools/ci_tools/dependency_analysis.py +++ b/tools/azure-sdk-tools/ci_tools/dependency_analysis.py @@ -58,22 +58,24 @@ def record_dep(dependencies: Dict[str, Dict[str, Any]], req_name: str, spec: str def get_lib_deps(base_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]: packages = {} dependencies = {} - for lib_dir in discover_targeted_packages("azure*", base_dir): - parsed = ParsedSetup.from_path(lib_dir) - lib_name, version, requires = parsed.name, parsed.version, parsed.requires + try: + parsed = ParsedSetup.from_path(lib_dir) + lib_name, version, requires = parsed.name, parsed.version, parsed.requires - packages[lib_name] = {"version": version, "source": lib_dir, "deps": []} + packages[lib_name] = {"version": version, "source": lib_dir, "deps": []} - for req in requires: - req_obj = parse_require(req) - req_name = req_obj.name - spec = req_obj.specifier if len(req_obj.specifier) else None - if spec is None: - spec = "" + for req in requires: + req_obj = parse_require(req) + req_name = req_obj.name + spec = req_obj.specifier if len(req_obj.specifier) else None + if spec is None: + spec = "" - packages[lib_name]["deps"].append({"name": req_name, "version": str(spec)}) - record_dep(dependencies, req_name, str(spec), lib_name) + packages[lib_name]["deps"].append({"name": req_name, "version": str(spec)}) + record_dep(dependencies, req_name, str(spec), lib_name) + except: + print(f"Failed to parse setup.py or pyproject.toml at {lib_dir}") return packages, dependencies