diff --git a/common/smoketest/dependencies.py b/common/smoketest/dependencies.py new file mode 100644 index 000000000000..eabc25e77cf5 --- /dev/null +++ b/common/smoketest/dependencies.py @@ -0,0 +1,71 @@ +import argparse +import pkg_resources + +try: + # pip < 20 + from pip._internal.req import parse_requirements + from pip._internal.download import PipSession +except: + # pip >= 20 + from pip._internal.req import parse_requirements + from pip._internal.network.session import PipSession + +def combine_requirements(requirements): + name = requirements[0].project_name + specs = [] + for req in requirements: + if len(req.specs) == 0: + continue + + specs.extend([s[0] + s[1] for s in req.specs]) + + return name + ",".join(specs) + +def get_dependencies(packages): + requirements = [] + for package in packages: + package_info = pkg_resources.working_set.by_key[package] + + applicable_requirements = [r for r in package_info.requires() if r.marker is None or r.marker.evaluate()] + requirements.extend(applicable_requirements) + + return requirements + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="List dependencies for a given requirements.txt file" + ) + + parser.add_argument( + "-r", + "--requirements", + dest="requirements_file", + help="File containing list of packages for which to find dependencies", + required=True + ) + + args = parser.parse_args() + # Get package names from requirements.txt + requirements = parse_requirements(args.requirements_file, session=PipSession()) + package_names = [item.req.name for item in requirements] + + dependencies = get_dependencies(package_names) + + # It may be the case that packages have multiple sets of dependency + # requirements, for example: + # Package A requires Foo>=1.0.0,<2.0.0 + # Package B requires Foo>=1.0.0,<1.2.3 + # This combines all required versions into one string for pip to resolve + # Output: Foo>=1.0.0,<2.0.0,>=1.0.0,<1.2.3 + # Pip parses this value using the Requirement object (https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirement-objects) + # According to https://packaging.python.org/glossary/#term-requirement-specifier + grouped_dependencies = {} + for dep in dependencies: + if dep.key in grouped_dependencies: + grouped_dependencies[dep.key].append(dep) + else: + grouped_dependencies[dep.key] = [dep] + + final_dependencies = [combine_requirements(r) for r in grouped_dependencies.values()] + + print("\n".join(final_dependencies)) diff --git a/common/smoketest/smoke-test.yml b/common/smoketest/smoke-test.yml index 6775f045dcad..0df4c50d35f6 100644 --- a/common/smoketest/smoke-test.yml +++ b/common/smoketest/smoke-test.yml @@ -84,18 +84,21 @@ jobs: pip --version displayName: pip --version - - script: pip install packaging pkginfo - displayName: Install requirements for dev tools - - - script: pip install -r ./common/smoketest/requirements.txt - displayName: "Install requirements.txt" + - script: pip install -r ./common/smoketest/requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple + displayName: Install requirements from dev feed without dependencies - script: pip install -r ./common/smoketest/requirements_async.txt displayName: "Install requirements_async.txt" condition: and(succeeded(), eq(variables['InstallAsyncRequirements'], 'true')) - - script: python ./eng/tox/install_dev_build_dependency.py -r ./common/smoketest/requirements.txt - displayName: "Install dev dependencies from feed" + - script: python ./common/smoketest/dependencies.py -r ./common/smoketest/requirements.txt | tee ./common/smoketest/requirements_dependencies.txt + displayName: Create dependency list from installed dev packages + + - script: pip install -r ./common/smoketest/requirements_dependencies.txt + displayName: Install dev package dependencies from PyPI + + - script: pip freeze + displayName: Show installed packages (pip freeze) - template: /eng/common/TestResources/deploy-test-resources.yml parameters: @@ -105,7 +108,7 @@ jobs: ArmTemplateParameters: $(ArmTemplateParameters) - script: python ./common/smoketest/program.py - displayName: "Run Smoke Test" + displayName: Run Smoke Test - template: /eng/common/TestResources/remove-test-resources.yml parameters: diff --git a/eng/tox/install_dev_build_dependency.py b/eng/tox/install_dev_build_dependency.py index afb0d374dfe6..df9eb28b9686 100644 --- a/eng/tox/install_dev_build_dependency.py +++ b/eng/tox/install_dev_build_dependency.py @@ -15,16 +15,6 @@ from pip._internal.operations import freeze -try: - # pip < 20 - from pip._internal.req import parse_requirements - from pip._internal.download import PipSession -except: - # pip >= 20 - from pip._internal.req import parse_requirements - from pip._internal.network.session import PipSession - - # import common_task module root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", "..")) common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks")) @@ -124,36 +114,15 @@ def install_dev_build_packages(pkg_name_to_exclude): "--target", dest="target_package", help="The target package directory on disk.", - required=False, - ) - - parser.add_argument( - "-r", - "--requirements", - dest="requirements_file", - help="Use dev builds of all installed azure-* packages", - required=False + required=True, ) args = parser.parse_args() - if not args.target_package and not args.requirements_file: - raise "Must specify -t or -r" - if args.target_package: # get target package name from target package path pkg_dir = path.abspath(args.target_package) pkg_name, _, ver = get_package_details(path.join(pkg_dir, "setup.py")) install_dev_build_packages(pkg_name) - elif args.requirements_file: - # Get package names from requirements.txt - requirements = parse_requirements(args.requirements_file, session=PipSession()) - package_names = [item.req.name for item in requirements] - - # Remove existing packages (that came from the public feed) and install - # from dev feed - uninstall_packages(package_names) - install_packages(package_names) -