From 93694cc5e3c77a9f8019de4190607e3d994f38b7 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 May 2020 22:21:23 -0700 Subject: [PATCH 01/10] smoke test uses new workflow for installing dev feed packages and resolving dependencies --- common/smoketest/dependencies.py | 55 ++++++++++++++++++++++++++++++++ common/smoketest/smoke-test.yml | 13 +++++--- 2 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 common/smoketest/dependencies.py diff --git a/common/smoketest/dependencies.py b/common/smoketest/dependencies.py new file mode 100644 index 000000000000..03b67d533fcf --- /dev/null +++ b/common/smoketest/dependencies.py @@ -0,0 +1,55 @@ +import argparse +import pkg_resources + +try: + # pip < 20 + from pip._internal.req import parse_requirements + from pip._internal.download import PipSession +except: + # pip >= 20 + from pip._internal.req import parse_requirements + from pip._internal.network.session import PipSession + +def get_freeze_string(requirement): + output = [] + output.append(requirement.project_name) + spec_list = [s[0] + s[1] for s in requirement.specs] + + output.extend(','.join(spec_list)) + + return ''.join(output) + + +def get_dependencies(packages): + requirements = [] + for package in packages: + package_info = pkg_resources.working_set.by_key[package] + + applicable_requirements = [r for r in package_info.requires() if r.marker is None or r.marker.evaluate()] + requirements.extend(applicable_requirements) + + return [get_freeze_string(r) for r in requirements] + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="List dependencies for a given requirements.txt file" + ) + + parser.add_argument( + "-r", + "--requirements", + dest="requirements_file", + help="File containing list of packages for which to find dependencies", + required=True + ) + + args = parser.parse_args() + # Get package names from requirements.txt + requirements = parse_requirements(args.requirements_file, session=PipSession()) + package_names = [item.req.name for item in requirements] + + # Remove existing packages (that came from the public feed) and install + # from dev feed + dependencies = get_dependencies(package_names) + + print("\n".join(dependencies)) diff --git a/common/smoketest/smoke-test.yml b/common/smoketest/smoke-test.yml index 6775f045dcad..b7f447d059e8 100644 --- a/common/smoketest/smoke-test.yml +++ b/common/smoketest/smoke-test.yml @@ -87,15 +87,18 @@ jobs: - script: pip install packaging pkginfo displayName: Install requirements for dev tools - - script: pip install -r ./common/smoketest/requirements.txt - displayName: "Install requirements.txt" + - script: pip install -r .\common\smoketest\requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple + displayName: Install requirements from dev feed without dependencies - script: pip install -r ./common/smoketest/requirements_async.txt displayName: "Install requirements_async.txt" condition: and(succeeded(), eq(variables['InstallAsyncRequirements'], 'true')) - - script: python ./eng/tox/install_dev_build_dependency.py -r ./common/smoketest/requirements.txt - displayName: "Install dev dependencies from feed" + - script: python ./eng/tox/install_dev_build_dependency.py -r ./common/smoketest/requirements.txt > ./common/smoketest/requirements_dependencies.txt + displayName: Install packages from the dev feed + + - script: pip install -r ./common/smoketest/requirements_dependencies.txt + displayName: Install dev package dependencies from PyPI - template: /eng/common/TestResources/deploy-test-resources.yml parameters: @@ -105,7 +108,7 @@ jobs: ArmTemplateParameters: $(ArmTemplateParameters) - script: python ./common/smoketest/program.py - displayName: "Run Smoke Test" + displayName: Run Smoke Test - template: /eng/common/TestResources/remove-test-resources.yml parameters: From 9748c331ee9dce68567b03f2b52d7f24636625b7 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 May 2020 22:22:55 -0700 Subject: [PATCH 02/10] smoke test dependencies packaging and pkginfo are not required --- common/smoketest/smoke-test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/common/smoketest/smoke-test.yml b/common/smoketest/smoke-test.yml index b7f447d059e8..f01e6cb5702e 100644 --- a/common/smoketest/smoke-test.yml +++ b/common/smoketest/smoke-test.yml @@ -84,9 +84,6 @@ jobs: pip --version displayName: pip --version - - script: pip install packaging pkginfo - displayName: Install requirements for dev tools - - script: pip install -r .\common\smoketest\requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple displayName: Install requirements from dev feed without dependencies From 2e3576534a995d485fbb80f432ed50df9a78c8f6 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 May 2020 22:26:34 -0700 Subject: [PATCH 03/10] no backslashes --- common/smoketest/smoke-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/smoketest/smoke-test.yml b/common/smoketest/smoke-test.yml index f01e6cb5702e..b7f9d3158a1c 100644 --- a/common/smoketest/smoke-test.yml +++ b/common/smoketest/smoke-test.yml @@ -84,7 +84,7 @@ jobs: pip --version displayName: pip --version - - script: pip install -r .\common\smoketest\requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple + - script: pip install -r ./common/smoketest/requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple displayName: Install requirements from dev feed without dependencies - script: pip install -r ./common/smoketest/requirements_async.txt From 4f511b412f3b7567eab577655ae92d256be10bf3 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 May 2020 22:28:41 -0700 Subject: [PATCH 04/10] actually use the new dependencies script --- common/smoketest/smoke-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/smoketest/smoke-test.yml b/common/smoketest/smoke-test.yml index b7f9d3158a1c..6daaaa1abc7d 100644 --- a/common/smoketest/smoke-test.yml +++ b/common/smoketest/smoke-test.yml @@ -91,7 +91,7 @@ jobs: displayName: "Install requirements_async.txt" condition: and(succeeded(), eq(variables['InstallAsyncRequirements'], 'true')) - - script: python ./eng/tox/install_dev_build_dependency.py -r ./common/smoketest/requirements.txt > ./common/smoketest/requirements_dependencies.txt + - script: python ./common/smoketest/dependencies.py -r ./common/smoketest/requirements.txt > ./common/smoketest/requirements_dependencies.txt displayName: Install packages from the dev feed - script: pip install -r ./common/smoketest/requirements_dependencies.txt From b30829d80765eab5c81195138bcc555fb6f41f9f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 May 2020 22:29:40 -0700 Subject: [PATCH 05/10] add pip freeze to show installed packages --- common/smoketest/smoke-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/smoketest/smoke-test.yml b/common/smoketest/smoke-test.yml index 6daaaa1abc7d..291714d39311 100644 --- a/common/smoketest/smoke-test.yml +++ b/common/smoketest/smoke-test.yml @@ -97,6 +97,9 @@ jobs: - script: pip install -r ./common/smoketest/requirements_dependencies.txt displayName: Install dev package dependencies from PyPI + - script: pip freeze + displayName: Show installed packages (pip freeze) + - template: /eng/common/TestResources/deploy-test-resources.yml parameters: ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/' From 143a530e4c66750afa4525a72a69e54a44a9f74b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 May 2020 22:47:20 -0700 Subject: [PATCH 06/10] first cut at deduping dependencies --- common/smoketest/dependencies.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/smoketest/dependencies.py b/common/smoketest/dependencies.py index 03b67d533fcf..5584102804ae 100644 --- a/common/smoketest/dependencies.py +++ b/common/smoketest/dependencies.py @@ -28,7 +28,7 @@ def get_dependencies(packages): applicable_requirements = [r for r in package_info.requires() if r.marker is None or r.marker.evaluate()] requirements.extend(applicable_requirements) - return [get_freeze_string(r) for r in requirements] + return requirements if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -52,4 +52,7 @@ def get_dependencies(packages): # from dev feed dependencies = get_dependencies(package_names) - print("\n".join(dependencies)) + # TODO: Do a better job of merging these that doesn't squash earlier entries + final_dependencies = {d.key: get_freeze_string(d) for d in dependencies} + + print("\n".join(final_dependencies.values())) From a69d7c4707352b00bc185d0b88fc7ae2698d2649 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 May 2020 23:30:10 -0700 Subject: [PATCH 07/10] first cut at grouping dependencies --- common/smoketest/dependencies.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/common/smoketest/dependencies.py b/common/smoketest/dependencies.py index 5584102804ae..1283c26030f2 100644 --- a/common/smoketest/dependencies.py +++ b/common/smoketest/dependencies.py @@ -10,15 +10,16 @@ from pip._internal.req import parse_requirements from pip._internal.network.session import PipSession -def get_freeze_string(requirement): - output = [] - output.append(requirement.project_name) - spec_list = [s[0] + s[1] for s in requirement.specs] +def combine_requirements(requirements): + name = requirements[0].project_name + specs = [] + for req in requirements: + if len(req.specs) == 0: + continue - output.extend(','.join(spec_list)) - - return ''.join(output) + specs.extend([s[0] + s[1] for s in req.specs]) + return name + ",".join(specs) def get_dependencies(packages): requirements = [] @@ -52,7 +53,13 @@ def get_dependencies(packages): # from dev feed dependencies = get_dependencies(package_names) - # TODO: Do a better job of merging these that doesn't squash earlier entries - final_dependencies = {d.key: get_freeze_string(d) for d in dependencies} + grouped_dependencies = {} + for dep in dependencies: + if dep.key in grouped_dependencies: + grouped_dependencies[dep.key].append(dep) + else: + grouped_dependencies[dep.key] = [dep] + + final_dependencies = [combine_requirements(r) for r in grouped_dependencies.values()] - print("\n".join(final_dependencies.values())) + print("\n".join(final_dependencies)) From f1735ee7f5c834397f779f5bb229482b3794bcb9 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 May 2020 23:45:58 -0700 Subject: [PATCH 08/10] describe how dependency version specifiers are combined --- common/smoketest/dependencies.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/common/smoketest/dependencies.py b/common/smoketest/dependencies.py index 1283c26030f2..eabc25e77cf5 100644 --- a/common/smoketest/dependencies.py +++ b/common/smoketest/dependencies.py @@ -49,10 +49,16 @@ def get_dependencies(packages): requirements = parse_requirements(args.requirements_file, session=PipSession()) package_names = [item.req.name for item in requirements] - # Remove existing packages (that came from the public feed) and install - # from dev feed dependencies = get_dependencies(package_names) + # It may be the case that packages have multiple sets of dependency + # requirements, for example: + # Package A requires Foo>=1.0.0,<2.0.0 + # Package B requires Foo>=1.0.0,<1.2.3 + # This combines all required versions into one string for pip to resolve + # Output: Foo>=1.0.0,<2.0.0,>=1.0.0,<1.2.3 + # Pip parses this value using the Requirement object (https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirement-objects) + # According to https://packaging.python.org/glossary/#term-requirement-specifier grouped_dependencies = {} for dep in dependencies: if dep.key in grouped_dependencies: From 1d85d7493d2adc072eaf22c411c2087ecb753060 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 14 May 2020 00:00:44 -0700 Subject: [PATCH 09/10] remove requirements.txt work from install_dev_build_dependency.py --- eng/tox/install_dev_build_dependency.py | 33 +------------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/eng/tox/install_dev_build_dependency.py b/eng/tox/install_dev_build_dependency.py index afb0d374dfe6..df9eb28b9686 100644 --- a/eng/tox/install_dev_build_dependency.py +++ b/eng/tox/install_dev_build_dependency.py @@ -15,16 +15,6 @@ from pip._internal.operations import freeze -try: - # pip < 20 - from pip._internal.req import parse_requirements - from pip._internal.download import PipSession -except: - # pip >= 20 - from pip._internal.req import parse_requirements - from pip._internal.network.session import PipSession - - # import common_task module root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", "..")) common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks")) @@ -124,36 +114,15 @@ def install_dev_build_packages(pkg_name_to_exclude): "--target", dest="target_package", help="The target package directory on disk.", - required=False, - ) - - parser.add_argument( - "-r", - "--requirements", - dest="requirements_file", - help="Use dev builds of all installed azure-* packages", - required=False + required=True, ) args = parser.parse_args() - if not args.target_package and not args.requirements_file: - raise "Must specify -t or -r" - if args.target_package: # get target package name from target package path pkg_dir = path.abspath(args.target_package) pkg_name, _, ver = get_package_details(path.join(pkg_dir, "setup.py")) install_dev_build_packages(pkg_name) - elif args.requirements_file: - # Get package names from requirements.txt - requirements = parse_requirements(args.requirements_file, session=PipSession()) - package_names = [item.req.name for item in requirements] - - # Remove existing packages (that came from the public feed) and install - # from dev feed - uninstall_packages(package_names) - install_packages(package_names) - From e5048b8bedd056d258ac4e82ea7d62703202052b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 14 May 2020 00:04:38 -0700 Subject: [PATCH 10/10] better task name, output requirements_dependencies.txt --- common/smoketest/smoke-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/smoketest/smoke-test.yml b/common/smoketest/smoke-test.yml index 291714d39311..0df4c50d35f6 100644 --- a/common/smoketest/smoke-test.yml +++ b/common/smoketest/smoke-test.yml @@ -91,8 +91,8 @@ jobs: displayName: "Install requirements_async.txt" condition: and(succeeded(), eq(variables['InstallAsyncRequirements'], 'true')) - - script: python ./common/smoketest/dependencies.py -r ./common/smoketest/requirements.txt > ./common/smoketest/requirements_dependencies.txt - displayName: Install packages from the dev feed + - script: python ./common/smoketest/dependencies.py -r ./common/smoketest/requirements.txt | tee ./common/smoketest/requirements_dependencies.txt + displayName: Create dependency list from installed dev packages - script: pip install -r ./common/smoketest/requirements_dependencies.txt displayName: Install dev package dependencies from PyPI