Beginning of SDK-Tools refactor

.gitignore

@@ -45,6 +45,7 @@ coverage-*.xml
 stderr.txt
 stdout.txt
 _proxy_log_*.log
+_azci_*.log
 
 # tox environment folders
 .tox/

scripts/devops_tasks/common_tasks.py

@@ -32,60 +32,11 @@
 
 DEV_REQ_FILE = "dev_requirements.txt"
 NEW_DEV_REQ_FILE = "new_dev_requirements.txt"
-NEW_REQ_PACKAGES = ["azure-core", "azure-mgmt-core"]
+
 
 logging.getLogger().setLevel(logging.INFO)
 
-OMITTED_CI_PACKAGES = [
-    "azure-mgmt-documentdb",
-    "azure-servicemanagement-legacy",
-    "azure-mgmt-scheduler",
-    "azure",
-    "azure-mgmt",
-    "azure-storage",
-    "azure-monitor",
-    "azure-mgmt-regionmove",
-]
-MANAGEMENT_PACKAGE_IDENTIFIERS = [
-    "mgmt",
-    "azure-cognitiveservices",
-    "azure-servicefabric",
-    "nspkg",
-    "azure-keyvault",
-    "azure-synapse",
-    "azure-ai-anomalydetector",
-]
-META_PACKAGES = ["azure", "azure-mgmt", "azure-keyvault"]
-REGRESSION_EXCLUDED_PACKAGES = [
-    "azure-common",
-]
-
-MANAGEMENT_PACKAGES_FILTER_EXCLUSIONS = [
-    "azure-mgmt-core",
-]
-
-TEST_COMPATIBILITY_MAP = {"azure-core-tracing-opentelemetry": "<3.10"}
-
-omit_regression = (
-    lambda x: "nspkg" not in x
-    and "mgmt" not in x
-    and os.path.basename(x) not in MANAGEMENT_PACKAGE_IDENTIFIERS
-    and os.path.basename(x) not in META_PACKAGES
-    and os.path.basename(x) not in REGRESSION_EXCLUDED_PACKAGES
-)
-omit_docs = lambda x: "nspkg" not in x and os.path.basename(x) not in META_PACKAGES
-omit_build = lambda x: x  # Dummy lambda to match omit type
-lambda_filter_azure_pkg = lambda x: x.startswith("azure") and "-nspkg" not in x
-omit_mgmt = lambda x: "mgmt" not in x or os.path.basename(x) in MANAGEMENT_PACKAGES_FILTER_EXCLUSIONS
-
-
-# dict of filter type and filter function
-omit_funct_dict = {
-    "Build": omit_build,
-    "Docs": omit_docs,
-    "Regression": omit_regression,
-    "Omit_management": omit_mgmt,
-}
+
 
 
 def log_file(file_location, is_error=False):
@@ -137,152 +88,6 @@ def str_to_bool(input_string):
         return False
 
 
-def parse_setup(setup_path):
-    setup_filename = os.path.join(setup_path, "setup.py")
-    mock_setup = textwrap.dedent(
-        """\
-    def setup(*args, **kwargs):
-        __setup_calls__.append((args, kwargs))
-    """
-    )
-    parsed_mock_setup = ast.parse(mock_setup, filename=setup_filename)
-    with io.open(setup_filename, "r", encoding="utf-8-sig") as setup_file:
-        parsed = ast.parse(setup_file.read())
-        for index, node in enumerate(parsed.body[:]):
-            if (
-                not isinstance(node, ast.Expr)
-                or not isinstance(node.value, ast.Call)
-                or not hasattr(node.value.func, "id")
-                or node.value.func.id != "setup"
-            ):
-                continue
-            parsed.body[index:index] = parsed_mock_setup.body
-            break
-
-    fixed = ast.fix_missing_locations(parsed)
-    codeobj = compile(fixed, setup_filename, "exec")
-    local_vars = {}
-    global_vars = {"__setup_calls__": []}
-    current_dir = os.getcwd()
-    working_dir = os.path.dirname(setup_filename)
-    os.chdir(working_dir)
-    exec(codeobj, global_vars, local_vars)
-    os.chdir(current_dir)
-    _, kwargs = global_vars["__setup_calls__"][0]
-
-    try:
-        python_requires = kwargs["python_requires"]
-    # most do not define this, fall back to what we define as universal
-    except KeyError as e:
-        python_requires = ">=2.7"
-
-    version = kwargs["version"]
-    name = kwargs["name"]
-
-    requires = []
-    if "install_requires" in kwargs:
-        requires = kwargs["install_requires"]
-
-    return name, version, python_requires, requires
-
-
-def parse_requirements_file(file_location):
-    with open(file_location, "r") as f:
-        reqs = f.read()
-
-    return dict((req.name, req) for req in parse_requirements(reqs))
-
-
-def parse_setup_requires(setup_path):
-    _, _, python_requires, _ = parse_setup(setup_path)
-
-    return python_requires
-
-
-def get_name_from_specifier(version):
-    return re.split(r"[><=]", version)[0]
-
-
-def filter_for_compatibility(package_set):
-    collected_packages = []
-    v = sys.version_info
-    running_major_version = Version(".".join([str(v[0]), str(v[1]), str(v[2])]))
-
-    for pkg in package_set:
-        spec_set = SpecifierSet(parse_setup_requires(pkg))
-
-        if running_major_version in spec_set:
-            collected_packages.append(pkg)
-
-    return collected_packages
-
-
-def compare_python_version(version_spec):
-    current_sys_version = parse(platform.python_version())
-    spec_set = SpecifierSet(version_spec)
-
-    return current_sys_version in spec_set
-
-
-def filter_packages_by_compatibility_override(package_set, resolve_basename=True):
-    return [
-        p
-        for p in package_set
-        if compare_python_version(TEST_COMPATIBILITY_MAP.get(os.path.basename(p) if resolve_basename else p, ">=2.7"))
-    ]
-
-
-# this function is where a glob string gets translated to a list of packages
-# It is called by both BUILD (package) and TEST. In the future, this function will be the central location
-# for handling targeting of release packages
-def process_glob_string(
-    glob_string,
-    target_root_dir,
-    additional_contains_filter="",
-    filter_type="Build",
-):
-    if glob_string:
-        individual_globs = glob_string.split(",")
-    else:
-        individual_globs = "azure-*"
-    collected_top_level_directories = []
-
-    for glob_string in individual_globs:
-        globbed = glob.glob(os.path.join(target_root_dir, glob_string, "setup.py")) + glob.glob(
-            os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py")
-        )
-        collected_top_level_directories.extend([os.path.dirname(p) for p in globbed])
-
-    # dedup, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*"
-    collected_directories = list(set([p for p in collected_top_level_directories if additional_contains_filter in p]))
-
-    # if we have individually queued this specific package, it's obvious that we want to build it specifically
-    # in this case, do not honor the omission list
-    if len(collected_directories) == 1:
-        pkg_set_ci_filtered = filter_for_compatibility(collected_directories)
-    # however, if there are multiple packages being built, we should honor the omission list and NOT build the omitted
-    # packages
-    else:
-        allowed_package_set = remove_omitted_packages(collected_directories)
-        pkg_set_ci_filtered = filter_for_compatibility(allowed_package_set)
-
-    # Apply filter based on filter type. for e.g. Docs, Regression, Management
-    pkg_set_ci_filtered = list(filter(omit_funct_dict.get(filter_type, omit_build), pkg_set_ci_filtered))
-    logging.info("Target packages after filtering by CI: {}".format(pkg_set_ci_filtered))
-    logging.info(
-        "Package(s) omitted by CI filter: {}".format(list(set(collected_directories) - set(pkg_set_ci_filtered)))
-    )
-    return sorted(pkg_set_ci_filtered)
-
-
-def remove_omitted_packages(collected_directories):
-    packages = [
-        package_dir for package_dir in collected_directories if os.path.basename(package_dir) not in OMITTED_CI_PACKAGES
-    ]
-
-    return packages
-
-
 def run_check_call(
     command_array,
     working_directory,
@@ -341,19 +146,6 @@ def is_error_code_5_allowed(target_pkg, pkg_name):
     else:
         return False
 
-
-def parse_require(req):
-    """
-    Parses the incoming version specification and returns a tuple of the requirement name and specifier.
-
-    "azure-core<2.0.0,>=1.11.0" -> [azure-core, <2.0.0,>=1.11.0]
-    """
-    req_object = Requirement.parse(req.split(";")[0])
-    pkg_name = req_object.key
-    spec = SpecifierSet(str(req_object).replace(pkg_name, ""))
-    return [pkg_name, spec]
-
-
 def find_whl(package_name, version, whl_directory):
     if not os.path.exists(whl_directory):
         logging.error("Whl directory is incorrect")
@@ -380,7 +172,6 @@ def find_whl(package_name, version, whl_directory):
 
     return whls[0]
 
-
 # This method installs package from a pre-built whl
 def install_package_from_whl(package_whl_path, working_dir, python_sym_link=sys.executable):
     commands = [
@@ -486,9 +277,3 @@ def get_installed_packages(paths=None):
     ws = WorkingSet(paths) if paths else working_set
     return ["{0}=={1}".format(p.project_name, p.version) for p in ws]
 
-
-def get_package_properties(setup_py_path):
-    """Parse setup.py and return package details like package name, version, whether it's new SDK"""
-    pkgName, version, _, requires = parse_setup(setup_py_path)
-    is_new_sdk = pkgName in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x)[0] in NEW_REQ_PACKAGES), requires))
-    return pkgName, version, is_new_sdk, setup_py_path

tools/azure-sdk-tools/azci/__init__.py

@@ -0,0 +1,3 @@
+from .dispatch import main
+
+__all__ = ["main"]

tools/azure-sdk-tools/azci/build.py

@@ -0,0 +1,6 @@
+from .run_configuration import RunConfiguration
+import pdb
+
+def build_action(config: RunConfiguration, args: Namespace) -> None:
+    pdb.set_trace()
+    pass