From 6704658a2a2781ba6f82a4658b89df513f9582e7 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 16 Sep 2019 11:56:21 -0700 Subject: [PATCH 1/2] originally removed this method due to a 'brilliant simplification' only to realize that kinda missed an essential filter during a build for a release --- eng/pipelines/templates/steps/build-artifacts.yml | 2 +- scripts/devops_tasks/common_tasks.py | 4 ++-- scripts/devops_tasks/setup_execute_tests.py | 11 ++++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/steps/build-artifacts.yml b/eng/pipelines/templates/steps/build-artifacts.yml index 26028e320d9e..e859eb029fcf 100644 --- a/eng/pipelines/templates/steps/build-artifacts.yml +++ b/eng/pipelines/templates/steps/build-artifacts.yml @@ -22,7 +22,7 @@ steps: displayName: 'Generate Python2 Applicable Namespace Packages' inputs: scriptPath: 'scripts/devops_tasks/build_packages.py' - arguments: '-d "$(Build.ArtifactStagingDirectory)" "*-nspkg" --service=${{parameters.ServiceDirectory}}' + arguments: '-d "$(Build.ArtifactStagingDirectory)" "${{ parameters.BuildTargetingString }}" --pkgfilter="nspkg" --service=${{parameters.ServiceDirectory}}' - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index fbe903704fcf..8b71885eafde 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -131,7 +131,7 @@ def filter_for_compatibility(package_set): # this function is where a glob string gets translated to a list of packages # It is called by both BUILD (package) and TEST. In the future, this function will be the central location # for handling targeting of release packages -def process_glob_string(glob_string, target_root_dir): +def process_glob_string(glob_string, target_root_dir, additional_contains_filter=""): if glob_string: individual_globs = glob_string.split(",") else: @@ -145,7 +145,7 @@ def process_glob_string(glob_string, target_root_dir): collected_top_level_directories.extend([os.path.dirname(p) for p in globbed]) # dedup, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*" - collected_directories = list(set(collected_top_level_directories)) + collected_directories = list(set([p for p in collected_top_level_directories if additional_contains_filter in p])) # if we have individually queued this specific package, it's obvious that we want to build it specifically # in this case, do not honor the omission list diff --git a/scripts/devops_tasks/setup_execute_tests.py b/scripts/devops_tasks/setup_execute_tests.py index 6ec2301fccdc..511e000a89ad 100644 --- a/scripts/devops_tasks/setup_execute_tests.py +++ b/scripts/devops_tasks/setup_execute_tests.py @@ -185,6 +185,15 @@ def execute_global_install_and_test( ), ) + parser.add_argument( + "--pkgfilter", + default="", + dest="package_filter_string", + help=( + "An additional string used to filter the set of artifacts by a simple CONTAINS clause. This filters packages AFTER the set is built with compatibility and omission lists accounted." + ), + ) + parser.add_argument( "--junitxml", dest="test_results", @@ -250,7 +259,7 @@ def execute_global_install_and_test( else: target_dir = root_dir - targeted_packages = process_glob_string(args.glob_string, target_dir) + targeted_packages = process_glob_string(args.glob_string, target_dir, args.package_filter_string) extended_pytest_args = [] if len(targeted_packages) == 0: From 47fecebc7e8f3221f26d091505a05c4df8d2b99d Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 16 Sep 2019 12:05:46 -0700 Subject: [PATCH 2/2] update proper file --- scripts/devops_tasks/build_packages.py | 11 ++++++++++- scripts/devops_tasks/setup_execute_tests.py | 11 +---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/devops_tasks/build_packages.py b/scripts/devops_tasks/build_packages.py index dc3a205336e2..2cd5dab475d1 100644 --- a/scripts/devops_tasks/build_packages.py +++ b/scripts/devops_tasks/build_packages.py @@ -64,6 +64,15 @@ def build_packages(targeted_packages, distribution_directory): ), ) + parser.add_argument( + "--pkgfilter", + default="", + dest="package_filter_string", + help=( + "An additional string used to filter the set of artifacts by a simple CONTAINS clause. This filters packages AFTER the set is built with compatibility and omission lists accounted." + ), + ) + args = parser.parse_args() # We need to support both CI builds of everything and individual service @@ -74,5 +83,5 @@ def build_packages(targeted_packages, distribution_directory): else: target_dir = root_dir - targeted_packages = process_glob_string(args.glob_string, target_dir) + targeted_packages = process_glob_string(args.glob_string, target_dir, args.package_filter_string) build_packages(targeted_packages, args.distribution_directory) diff --git a/scripts/devops_tasks/setup_execute_tests.py b/scripts/devops_tasks/setup_execute_tests.py index 511e000a89ad..6ec2301fccdc 100644 --- a/scripts/devops_tasks/setup_execute_tests.py +++ b/scripts/devops_tasks/setup_execute_tests.py @@ -185,15 +185,6 @@ def execute_global_install_and_test( ), ) - parser.add_argument( - "--pkgfilter", - default="", - dest="package_filter_string", - help=( - "An additional string used to filter the set of artifacts by a simple CONTAINS clause. This filters packages AFTER the set is built with compatibility and omission lists accounted." - ), - ) - parser.add_argument( "--junitxml", dest="test_results", @@ -259,7 +250,7 @@ def execute_global_install_and_test( else: target_dir = root_dir - targeted_packages = process_glob_string(args.glob_string, target_dir, args.package_filter_string) + targeted_packages = process_glob_string(args.glob_string, target_dir) extended_pytest_args = [] if len(targeted_packages) == 0: