diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 78b62ba3d7d5c..d61876ee17f9f 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -86,7 +86,6 @@ ("NOTICE", "/opt/airflow/NOTICE"), ("RELEASE_NOTES.rst", "/opt/airflow/RELEASE_NOTES.rst"), ("airflow", "/opt/airflow/airflow"), - ("provider_packages", "/opt/airflow/provider_packages"), ("dags", "/opt/airflow/dags"), ("dev", "/opt/airflow/dev"), ("docs", "/opt/airflow/docs"), @@ -95,6 +94,7 @@ ("images", "/opt/airflow/images"), ("logs", "/root/airflow/logs"), ("pyproject.toml", "/opt/airflow/pyproject.toml"), + ("providers", "/opt/airflow/providers"), ("pytest.ini", "/opt/airflow/pytest.ini"), ("scripts", "/opt/airflow/scripts"), ("scripts/docker/entrypoint_ci.sh", "/entrypoint"), diff --git a/dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2 b/dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2 deleted file mode 100644 index 1cbbab4513776..0000000000000 --- a/dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2 +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE -# OVERWRITTEN WHEN PREPARING PACKAGES. - -# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE -# `MANIFEST_TEMPLATE.py.jinja2` IN the `provider_packages` DIRECTORY - - -{% if PROVIDER_PACKAGE_ID == 'amazon' %} -include airflow/providers/amazon/aws/hooks/batch_waiters.json -{% elif PROVIDER_PACKAGE_ID == 'google' %} -include airflow/providers/google/cloud/example_dags/*.yaml -include airflow/providers/google/cloud/example_dags/*.sql -{% elif PROVIDER_PACKAGE_ID == 'cncf.kubernetes' %} -include airflow/providers/cncf/kubernetes/*.jinja2 -{% endif %} - -include NOTICE -include LICENSE -include CHANGELOG.txt -include README.md -global-exclude __pycache__ *.pyc diff --git a/dev/provider_packages/move_providers.py b/dev/provider_packages/move_providers.py new file mode 100755 index 0000000000000..7d3d868575f10 --- /dev/null +++ b/dev/provider_packages/move_providers.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import json +import os +import re +import shutil +from collections import defaultdict +from dataclasses import asdict, dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Any + +import jsonschema +import yaml +from jinja2 import Template +from rich.console import Console + +console = Console(width=400, color_system="standard") + +ALL_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] + +AIRFLOW_SOURCES_ROOT_PATH = Path(__file__).parents[2].resolve() + +TEMPLATES_DIR = Path(__file__).parent / "templates" + +PYPROJECT_TOML_TEMPLATE_FILE = TEMPLATES_DIR / "PYPROJECT_TEMPLATE.toml.jinja2" +GET_PROVIDER_INFO_TEMPLATE_FILE = TEMPLATES_DIR / "get_provider_info_TEMPLATE.py.jinja2" +README_TEMPLATE_FILE = TEMPLATES_DIR / "PROVIDER_README_TEMPLATE.rst.jinja2" + +ORIGIN_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT_PATH / "airflow" / "providers" +ORIGIN_UNIT_TESTS_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT_PATH / "tests" / "providers" +ORIGIN_INTEGRATION_TESTS_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT_PATH / "tests" / "integration" / "providers" +ORIGIN_SYSTEM_TESTS_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT_PATH / "tests" / "system" / "providers" +ORIGIN_DOCS_DIR = AIRFLOW_SOURCES_ROOT_PATH / "docs" +ORIGIN_SYSTEM_CONFTEST_PY_FILE_PATH = AIRFLOW_SOURCES_ROOT_PATH / "tests" / "system" / "conftest.py" + +TARGET_PROVIDER_DIR = AIRFLOW_SOURCES_ROOT_PATH / "providers" +PROVIDER_DEPENDENCIES_JSON_FILE = AIRFLOW_SOURCES_ROOT_PATH / "generated" / "provider_dependencies.json" +PROVIDER_RUNTIME_DATA_SCHEMA_PATH = AIRFLOW_SOURCES_ROOT_PATH / "airflow" / "provider_info.schema.json" +PROVIDER_DEPENDENCIES = json.loads(PROVIDER_DEPENDENCIES_JSON_FILE.read_text()) + +LICENSE_RST = """ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +""" + + +@dataclass() +class ProviderMoveInfo: + source_provider_path: Path + provider_relative_path: Path = field(init=False) + provider_id: str = field(init=False) + origin_provider_unit_tests_path: Path = field(init=False) + origin_provider_integration_tests_path: Path = field(init=False) + origin_provider_system_tests_path: Path = field(init=False) + origin_docs_dir: Path = field(init=False) + target_provider_path: Path = field(init=False) + target_provider_package_sources_path: Path = field(init=False) + target_provider_unit_tests_path: Path = field(init=False) + target_provider_integration_tests_path: Path = field(init=False) + target_provider_system_tests_path: Path = field(init=False) + target_pyproject_toml_file: Path = field(init=False) + target_provider_yaml_file: Path = field(init=False) + target_provider_readme_file: Path = field(init=False) + target_get_provider_info_file: Path = field(init=False) + target_docs_dir: Path = field(init=False) + target_changelog_file: Path = field(init=False) + + def __post_init__(self): + self.provider_relative_path = self.source_provider_path.relative_to(ORIGIN_PROVIDERS_DIR) + self.provider_id = self.provider_relative_path.as_posix().replace("/", ".") + self.origin_provider_unit_tests_path = ORIGIN_UNIT_TESTS_PROVIDERS_DIR / self.provider_relative_path + self.origin_provider_integration_tests_path = ( + ORIGIN_INTEGRATION_TESTS_PROVIDERS_DIR / self.provider_relative_path + ) + self.origin_provider_system_tests_path = ( + ORIGIN_SYSTEM_TESTS_PROVIDERS_DIR / self.provider_relative_path + ) + self.origin_docs_dir = ( + ORIGIN_DOCS_DIR / f"apache-airflow-providers-{self.provider_id.replace('.', '-')}" + ) + self.target_provider_path = TARGET_PROVIDER_DIR / self.source_provider_path.relative_to( + ORIGIN_PROVIDERS_DIR + ) + self.target_provider_package_sources_path = ( + self.target_provider_path / "src" / "airflow" / "providers" / self.provider_relative_path + ) + self.target_provider_unit_tests_path = ( + self.target_provider_path / "tests" / "airflow" / "providers" / self.provider_relative_path + ) + self.target_provider_integration_tests_path = ( + self.target_provider_path + / "tests" + / "integration" + / "airflow" + / "providers" + / self.provider_relative_path + ) + self.target_provider_system_tests_path = ( + self.target_provider_path + / "tests" + / "system" + / "airflow" + / "providers" + / self.provider_relative_path + ) + self.target_pyproject_toml_file = self.target_provider_path / "pyproject.toml" + self.target_provider_yaml_file = self.target_provider_path / "provider.yaml" + self.target_provider_readme_file = self.target_provider_path / "README.rst" + self.target_get_provider_info_file = ( + self.target_provider_package_sources_path / "get_provider_info.py" + ) + self.target_docs_dir = self.target_provider_path / "docs" + self.target_changelog_file = self.target_docs_dir / "CHANGELOG.rst" + + def get_dict(self): + _dict_repr = asdict(self) + for key, value in _dict_repr.items(): + if isinstance(value, Path): + _dict_repr[key] = os.fspath(value) + return _dict_repr + + +def validate_provider_info_with_runtime_schema(provider_info: dict[str, Any]) -> None: + """ + Validates provider info against the runtime schema. This way we check if the provider info in the + packages is future-compatible. The Runtime Schema should only change when there is a major version + change. + + :param provider_info: provider info to validate + """ + + with open(PROVIDER_RUNTIME_DATA_SCHEMA_PATH) as schema_file: + schema = json.load(schema_file) + try: + jsonschema.validate(provider_info, schema=schema) + except jsonschema.ValidationError as ex: + raise Exception( + "Error when validating schema. The schema must be compatible with " + "airflow/provider_info.schema.json.", + ex, + ) + + +def black_format(content) -> str: + from black import format_str + + return format_str(content, mode=black_mode()) + + +@lru_cache(maxsize=None) +def black_mode(): + from black import Mode, parse_pyproject_toml, target_version_option_callback + + config = parse_pyproject_toml(os.path.join(AIRFLOW_SOURCES_ROOT_PATH, "pyproject.toml")) + + target_versions = set( + target_version_option_callback(None, None, tuple(config.get("target_version", ()))), + ) + + return Mode( + target_versions=target_versions, + line_length=config.get("line_length", Mode.line_length), + is_pyi=bool(config.get("is_pyi", Mode.is_pyi)), + string_normalization=not bool(config.get("skip_string_normalization", not Mode.string_normalization)), + experimental_string_processing=bool( + config.get("experimental_string_processing", Mode.experimental_string_processing) + ), + ) + + +def find_providers_dirs() -> list[ProviderMoveInfo]: + providers_dirs = [] + for file in ORIGIN_PROVIDERS_DIR.rglob("provider.yaml"): + providers_dirs.append(ProviderMoveInfo(file.parent.resolve())) + return providers_dirs + + +def moving(source_path: Path, target_path: Path, create_if_not_exists: bool = True) -> None: + console.print(f"[bright_blue]Moving {source_path} -> {target_path}") + target_path.parent.mkdir(parents=True, exist_ok=True) + if not source_path.exists(): + if create_if_not_exists: + target_path.touch() + return + else: + raise Exception(f"Source path {source_path} does not exist!") + source_path.rename(target_path) + + +def move_provider_sources(move_info: ProviderMoveInfo): + console.print(f"[yellow]Moving provider: {move_info.provider_id}") + if move_info.target_provider_path.exists(): + raise Exception(f"Target directory {move_info.target_provider_path} already exists. Skipping.") + moving(move_info.source_provider_path, move_info.target_provider_package_sources_path) + if move_info.origin_provider_unit_tests_path.exists(): + moving(move_info.origin_provider_unit_tests_path, move_info.target_provider_unit_tests_path) + if move_info.origin_provider_integration_tests_path.exists(): + moving( + move_info.origin_provider_integration_tests_path, + move_info.target_provider_integration_tests_path, + ) + if move_info.origin_provider_system_tests_path.exists(): + moving(move_info.origin_provider_system_tests_path, move_info.target_provider_system_tests_path) + shutil.copy( + ORIGIN_SYSTEM_CONFTEST_PY_FILE_PATH, move_info.target_provider_system_tests_path / "conftest.py" + ) + + +def escape_list_of_dependencies(deps: list[str]) -> list[str]: + return [dependency.replace('"', '\\"') for dependency in deps] + + +def generate_pyproject_toml(move_info: ProviderMoveInfo): + console.print(f"[yellow]Generating {move_info.target_provider_yaml_file}") + provider_yaml_info = yaml.safe_load(move_info.target_provider_yaml_file.read_text()) + provider_yaml_info["supported_python_versions"] = get_supported_python_versions(provider_yaml_info) + optional_dependencies: dict[str, list[str]] = defaultdict(list) + cross_deps = PROVIDER_DEPENDENCIES[move_info.provider_id].get("cross-provider-deps") + if cross_deps: + for extra in cross_deps: + optional_dependencies[extra].append(f"apache-airflow-providers-{extra.replace('.', '-')}") + if provider_yaml_info.get("additional-extras"): + for extra in provider_yaml_info["additional-extras"]: + name = extra["name"] + escaped_deps = escape_list_of_dependencies(extra["dependencies"]) + for escape_dep in escaped_deps: + base_package = escape_dep.strip("<>=~") + if optional_dependencies.get(name): + # Remove automatically added non-versioned cross-dependency package if already present + optional_dependencies[name].remove(base_package) + optional_dependencies[name].append(escape_dep) + + provider_yaml_info["optional_dependencies"] = optional_dependencies + provider_yaml_info["package_id"] = move_info.provider_id + provider_yaml_info["package_name"] = provider_yaml_info["package-name"] + escaped_dependencies = escape_list_of_dependencies(provider_yaml_info["dependencies"]) + provider_yaml_info["escaped_dependencies"] = escaped_dependencies + template = Template(PYPROJECT_TOML_TEMPLATE_FILE.read_text()) + rendered = template.render(provider_yaml_info) + move_info.target_pyproject_toml_file.write_text(rendered) + + +def get_supported_python_versions(provider_info: dict[str, Any]) -> list[str]: + excluded_versions = provider_info.get("excluded_python_versions") + if not excluded_versions: + return ALL_PYTHON_VERSIONS + return [p for p in ALL_PYTHON_VERSIONS if p not in excluded_versions] + + +def generate_get_provider_info(move_info: ProviderMoveInfo): + console.print(f"[yellow]Generating {move_info.target_get_provider_info_file}") + provider_yaml_info = yaml.safe_load(move_info.target_provider_yaml_file.read_text()) + validate_provider_info_with_runtime_schema(provider_yaml_info) + context = {"PROVIDER_INFO": provider_yaml_info} + template = Template(GET_PROVIDER_INFO_TEMPLATE_FILE.read_text()) + rendered = template.render(context) + move_info.target_get_provider_info_file.write_text(black_format(rendered)) + + +def convert_pip_requirements_to_table(requirements: list[str]) -> str: + """ + Converts PIP requirement list to an RST table. + :param requirements: requirements list + :return: formatted table + """ + from tabulate import tabulate + + headers = ["PIP package", "Version required"] + table_data: list[tuple[str, str]] = [] + for dependency in requirements: + found = re.match(r"(^[^<=>~]*)([^<=>~]?.*)$", dependency) + if found: + package = found.group(1) + version_required = found.group(2) + if version_required != "": + version_required = f"``{version_required}``" + table_data.append((f"``{package}``", version_required)) + else: + table_data.append((dependency, "")) + return tabulate(table_data, headers=headers, tablefmt="rst") + + +def convert_cross_package_dependencies_to_table( + cross_package_dependencies: list[str], +) -> str: + """ + Converts cross-package dependencies to an RST table + :param cross_package_dependencies: list of cross-package dependencies + :return: formatted table + """ + from tabulate import tabulate + + headers = ["Dependent package", "Extra"] + table_data = [] + prefix = "apache-airflow-providers-" + base_url = "https://airflow.apache.org/docsdocs/" + for dependency in cross_package_dependencies: + pip_package_name = f"{prefix}{dependency.replace('.','-')}" + url_suffix = f"{dependency.replace('.','-')}" + url = f"`{pip_package_name} <{base_url}{prefix}{url_suffix}>`_" + table_data.append((url, f"``{dependency}``")) + return tabulate(table_data, headers=headers, tablefmt="rst") + + +def generate_readme_file(move_info: ProviderMoveInfo): + console.print(f"[yellow]Generating {move_info.target_provider_readme_file}") + provider_yaml_info = yaml.safe_load(move_info.target_provider_yaml_file.read_text()) + context = { + "PACKAGE_PIP_NAME": provider_yaml_info["package-name"], + "RELEASE": provider_yaml_info["versions"][0], + "FULL_PACKAGE_NAME": f"airflow.providers.{move_info.provider_id}", + "VERSION_SUFFIX": "", + "PROVIDER_DESCRIPTION": provider_yaml_info["description"], + "PROVIDER_PACKAGE_ID": move_info.provider_id, + "SUPPORTED_PYTHON_VERSIONS": get_supported_python_versions(provider_yaml_info), + "PIP_REQUIREMENTS": provider_yaml_info["dependencies"], + "PIP_REQUIREMENTS_TABLE_RST": convert_pip_requirements_to_table(provider_yaml_info["dependencies"]), + "CROSS_PROVIDERS_DEPENDENCIES": PROVIDER_DEPENDENCIES[move_info.provider_id].get( + "cross-provider-deps" + ), + "CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST": convert_cross_package_dependencies_to_table( + PROVIDER_DEPENDENCIES[move_info.provider_id] + ), + "CHANGELOG": move_info.target_changelog_file.read_text(), + } + template = Template(README_TEMPLATE_FILE.read_text()) + readme_content = LICENSE_RST + template.render(context) + move_info.target_provider_readme_file.write_text(readme_content) + + +def generate_other_files(move_info: ProviderMoveInfo): + context = yaml.safe_load(move_info.target_provider_yaml_file.read_text()) + context["package_name"] = context["package-name"] + for file_name in ["INSTALL.txt", "LICENSE.txt", "NOTICE.txt"]: + template = Template((TEMPLATES_DIR / file_name).with_suffix(".txt.jinja2").read_text()) + file_content = template.render(context) + (move_info.target_provider_path / file_name).write_text(file_content) + + +def move_provider(provider_info: ProviderMoveInfo): + console.print(provider_info.get_dict()) + move_provider_sources(provider_info) + moving(provider_info.origin_docs_dir, provider_info.target_docs_dir) + moving( + provider_info.target_provider_package_sources_path / "CHANGELOG.rst", + provider_info.target_changelog_file, + ) + moving( + provider_info.target_provider_package_sources_path / "provider.yaml", + provider_info.target_provider_yaml_file, + ) + moving( + provider_info.target_provider_package_sources_path / ".latest-doc-only-change.txt", + provider_info.target_provider_path / ".latest-doc-only-change.txt", + create_if_not_exists=True, + ) + generate_pyproject_toml(provider_info) + generate_get_provider_info(provider_info) + generate_readme_file(provider_info) + generate_other_files(provider_info) + + +def move_all_providers(): + for api_dir in ORIGIN_DOCS_DIR.rglob("_api"): + console.print(f"[yellow]Removing {api_dir}") + shutil.rmtree(api_dir) + shutil.rmtree(ORIGIN_DOCS_DIR / "_build", ignore_errors=True) + shutil.rmtree(ORIGIN_DOCS_DIR / "_doctrees", ignore_errors=True) + shutil.rmtree(ORIGIN_DOCS_DIR / "_inventory_cache", ignore_errors=True) + for info in find_providers_dirs(): + move_provider(provider_info=info) + + shutil.rmtree(ORIGIN_PROVIDERS_DIR, ignore_errors=True) + shutil.rmtree(ORIGIN_INTEGRATION_TESTS_PROVIDERS_DIR, ignore_errors=True) + shutil.rmtree(ORIGIN_SYSTEM_TESTS_PROVIDERS_DIR, ignore_errors=True) + ORIGIN_SYSTEM_CONFTEST_PY_FILE_PATH.unlink() + + +if __name__ == "__main__": + move_all_providers() diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 2ef0859c8980e..f08b0806f6611 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -22,12 +22,10 @@ import collections import difflib -import glob import json import logging import os import re -import shutil import subprocess import sys import tempfile @@ -36,8 +34,7 @@ from copy import deepcopy from datetime import datetime, timedelta from enum import Enum -from functools import lru_cache -from os.path import dirname, relpath +from os.path import dirname from pathlib import Path from random import choice from shutil import copyfile @@ -90,17 +87,14 @@ MY_DIR_PATH = Path(__file__).parent AIRFLOW_SOURCES_ROOT_PATH = MY_DIR_PATH.parents[1] +TEMPLATE_DIR_PATH = MY_DIR_PATH / "templates" AIRFLOW_PATH = AIRFLOW_SOURCES_ROOT_PATH / "airflow" DIST_PATH = AIRFLOW_SOURCES_ROOT_PATH / "dist" -PROVIDERS_PATH = AIRFLOW_PATH / "providers" +PROVIDERS_PATH = AIRFLOW_SOURCES_ROOT_PATH / "providers" DOCUMENTATION_PATH = AIRFLOW_SOURCES_ROOT_PATH / "docs" DEPENDENCIES_JSON_FILE_PATH = AIRFLOW_SOURCES_ROOT_PATH / "generated" / "provider_dependencies.json" -TARGET_PROVIDER_PACKAGES_PATH = AIRFLOW_SOURCES_ROOT_PATH / "provider_packages" -GENERATED_AIRFLOW_PATH = TARGET_PROVIDER_PACKAGES_PATH / "airflow" -GENERATED_PROVIDERS_PATH = GENERATED_AIRFLOW_PATH / "providers" - PROVIDER_RUNTIME_DATA_SCHEMA_PATH = AIRFLOW_SOURCES_ROOT_PATH / "airflow" / "provider_info.schema.json" CROSS_PROVIDERS_DEPS = "cross-providers-deps" @@ -130,8 +124,8 @@ class ProviderPackageDetails(NamedTuple): provider_package_id: str full_package_name: str pypi_package_name: str - source_provider_package_path: str - documentation_provider_package_path: str + source_provider_package_path: Path + documentation_provider_package_path: Path provider_description: str versions: list[str] excluded_python_versions: list[str] @@ -164,15 +158,6 @@ def cli(): help=f"If the git remote {HTTPS_REMOTE} already exists, don't try to update it", ) -option_package_format = click.option( - "--package-format", - type=click.Choice(["wheel", "sdist", "both"]), - help="Format of packages.", - default="wheel", - show_default=True, - envvar="PACKAGE_FORMAT", -) - option_version_suffix = click.option( "--version-suffix", metavar="suffix", @@ -402,7 +387,7 @@ def render_template( """ import jinja2 - template_loader = jinja2.FileSystemLoader(searchpath=MY_DIR_PATH) + template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR_PATH) template_env = jinja2.Environment( loader=template_loader, undefined=jinja2.StrictUndefined, @@ -780,59 +765,22 @@ def get_git_tag_check_command(tag: str) -> list[str]: ] -def get_source_package_path(provider_package_id: str) -> str: +def get_source_package_path(provider_package_id: str) -> Path: """ Retrieves source package path from package id. :param provider_package_id: id of the package :return: path of the providers folder """ - return os.path.join(PROVIDERS_PATH, *provider_package_id.split(".")) + return PROVIDERS_PATH / provider_package_id.replace(".", "/") -def get_documentation_package_path(provider_package_id: str) -> str: +def get_documentation_package_path(provider_package_id: str) -> Path: """ Retrieves documentation package path from package id. :param provider_package_id: id of the package :return: path of the documentation folder """ - return os.path.join( - DOCUMENTATION_PATH, f"apache-airflow-providers-{provider_package_id.replace('.','-')}" - ) - - -def get_generated_package_path(provider_package_id: str) -> str: - """ - Retrieves generated package path from package id. - :param provider_package_id: id of the package - :return: path of the providers folder - """ - provider_package_path = os.path.join(GENERATED_PROVIDERS_PATH, *provider_package_id.split(".")) - return provider_package_path - - -def get_additional_package_info(provider_package_path: str) -> str: - """ - Returns additional info for the package. - - :param provider_package_path: path for the package - :return: additional information for the path (empty string if missing) - """ - additional_info_file_path = os.path.join(provider_package_path, "ADDITIONAL_INFO.md") - if os.path.isfile(additional_info_file_path): - with open(additional_info_file_path) as additional_info_file: - additional_info = additional_info_file.read() - - additional_info_lines = additional_info.splitlines(keepends=True) - result = "" - skip_comment = True - for line in additional_info_lines: - if line.startswith(" -->"): - skip_comment = False - continue - if not skip_comment: - result += line - return result - return "" + return get_source_package_path(provider_package_id) / "docs" def get_package_pip_name(provider_package_id: str): @@ -869,7 +817,7 @@ def get_provider_yaml(provider_package_id: str) -> dict[str, Any]: :param provider_package_id: package id to retrieve provider.yaml from :return: provider_info dictionary """ - provider_yaml_file_name = os.path.join(get_source_package_path(provider_package_id), "provider.yaml") + provider_yaml_file_name = get_source_package_path(provider_package_id) / "provider.yaml" if not os.path.exists(provider_yaml_file_name): raise Exception(f"The provider.yaml file is missing: {provider_yaml_file_name}") with open(provider_yaml_file_name) as provider_file: @@ -931,10 +879,10 @@ def get_all_changes_for_package( ) if changes: provider_details = get_provider_details(provider_package_id) - doc_only_change_file = os.path.join( - provider_details.source_provider_package_path, ".latest-doc-only-change.txt" + doc_only_change_file = ( + provider_details.source_provider_package_path / ".latest-doc-only-change.txt" ) - if os.path.exists(doc_only_change_file): + if doc_only_change_file.exists(): with open(doc_only_change_file) as f: last_doc_only_hash = f.read().strip() try: @@ -1076,9 +1024,6 @@ def get_provider_jinja_context( "RELEASE": current_release_version, "RELEASE_NO_LEADING_ZEROS": release_version_no_leading_zeros, "VERSION_SUFFIX": version_suffix or "", - "ADDITIONAL_INFO": get_additional_package_info( - provider_package_path=provider_details.source_provider_package_path - ), "CROSS_PROVIDERS_DEPENDENCIES": cross_providers_dependencies, "PIP_REQUIREMENTS": get_provider_requirements(provider_details.provider_package_id), "PROVIDER_TYPE": "Provider", @@ -1094,9 +1039,8 @@ def get_provider_jinja_context( "PIP_REQUIREMENTS_TABLE": pip_requirements_table, "PIP_REQUIREMENTS_TABLE_RST": pip_requirements_table_rst, "PROVIDER_INFO": provider_info, - "CHANGELOG_RELATIVE_PATH": relpath( - provider_details.source_provider_package_path, - provider_details.documentation_provider_package_path, + "CHANGELOG_RELATIVE_PATH": provider_details.documentation_provider_package_path.relative_to( + provider_details.source_provider_package_path ), "CHANGELOG": changelog, "SUPPORTED_PYTHON_VERSIONS": supported_python_versions, @@ -1105,15 +1049,6 @@ def get_provider_jinja_context( return context -def prepare_readme_file(context): - readme_content = LICENCE_RST + render_template( - template_name="PROVIDER_README", context=context, extension=".rst" - ) - readme_file_path = os.path.join(TARGET_PROVIDER_PACKAGES_PATH, "README.rst") - with open(readme_file_path, "wt") as readme_file: - readme_file.write(readme_content) - - def confirm(message: str, answer: str | None = None) -> bool: """ Ask user to confirm (case-insensitive). @@ -1181,12 +1116,11 @@ def mark_latest_changes_as_documentation_only(provider_package_id: str, latest_c f"Marking last change: {latest_change.short_hash} and all above changes since the last release " "as doc-only changes!" ) - with open( - os.path.join(provider_details.source_provider_package_path, ".latest-doc-only-change.txt"), "tw" - ) as f: - f.write(latest_change.full_hash + "\n") - # exit code 66 marks doc-only change marked - sys.exit(66) + (provider_details.source_provider_package_path / ".latest-doc-only-change.txt").write_text( + latest_change.full_hash + "\n" + ) + # exit code 66 marks doc-only change + sys.exit(66) def add_new_version(type_of_change: TypeOfChange, provider_package_id: str): @@ -1199,7 +1133,7 @@ def add_new_version(type_of_change: TypeOfChange, provider_package_id: str): v = v.bump_minor() elif type_of_change == TypeOfChange.BUGFIX: v = v.bump_patch() - provider_yaml_path = Path(get_source_package_path(provider_package_id)) / "provider.yaml" + provider_yaml_path = get_source_package_path(provider_package_id) / "provider.yaml" original_text = provider_yaml_path.read_text() new_text = re.sub(r"versions:", f"versions:\n - {v}", original_text, 1) provider_yaml_path.write_text(new_text) @@ -1273,38 +1207,6 @@ def update_release_notes( return True -def update_setup_files( - provider_package_id: str, - version_suffix: str, -): - """ - Updates generated setup.cfg/setup.py/manifest.in/provider_info for packages - - :param provider_package_id: id of the package - :param version_suffix: version suffix corresponding to the version in the code - :returns False if the package should be skipped, True if everything generated properly - """ - verify_provider_package(provider_package_id) - provider_details = get_provider_details(provider_package_id) - provider_info = get_provider_info_from_provider_yaml(provider_package_id) - current_release_version = provider_details.versions[0] - jinja_context = get_provider_jinja_context( - provider_info=provider_info, - provider_details=provider_details, - current_release_version=current_release_version, - version_suffix=version_suffix, - ) - console.print() - console.print(f"Generating setup files for {provider_package_id}") - console.print() - prepare_setup_py_file(jinja_context) - prepare_setup_cfg_file(jinja_context) - prepare_get_provider_info_py_file(jinja_context, provider_package_id) - prepare_manifest_in_file(jinja_context) - prepare_readme_file(jinja_context) - return True - - def replace_content(file_path, old_text, new_text, provider_package_id): if new_text != old_text: _, temp_file_path = tempfile.mkstemp() @@ -1368,89 +1270,6 @@ def update_commits_rst( replace_content(index_file_path, old_text, new_text, provider_package_id) -@lru_cache(maxsize=None) -def black_mode(): - from black import Mode, parse_pyproject_toml, target_version_option_callback - - config = parse_pyproject_toml(os.path.join(AIRFLOW_SOURCES_ROOT_PATH, "pyproject.toml")) - - target_versions = set( - target_version_option_callback(None, None, tuple(config.get("target_version", ()))), - ) - - return Mode( - target_versions=target_versions, - line_length=config.get("line_length", Mode.line_length), - is_pyi=bool(config.get("is_pyi", Mode.is_pyi)), - string_normalization=not bool(config.get("skip_string_normalization", not Mode.string_normalization)), - experimental_string_processing=bool( - config.get("experimental_string_processing", Mode.experimental_string_processing) - ), - ) - - -def black_format(content) -> str: - from black import format_str - - return format_str(content, mode=black_mode()) - - -def prepare_setup_py_file(context): - setup_py_template_name = "SETUP" - setup_py_file_path = os.path.abspath(os.path.join(get_target_folder(), "setup.py")) - setup_py_content = render_template( - template_name=setup_py_template_name, context=context, extension=".py", autoescape=False - ) - with open(setup_py_file_path, "wt") as setup_py_file: - setup_py_file.write(black_format(setup_py_content)) - - -def prepare_setup_cfg_file(context): - setup_cfg_template_name = "SETUP" - setup_cfg_file_path = os.path.abspath(os.path.join(get_target_folder(), "setup.cfg")) - setup_cfg_content = render_template( - template_name=setup_cfg_template_name, - context=context, - extension=".cfg", - autoescape=False, - keep_trailing_newline=True, - ) - with open(setup_cfg_file_path, "wt") as setup_cfg_file: - setup_cfg_file.write(setup_cfg_content) - - -def prepare_get_provider_info_py_file(context, provider_package_id: str): - get_provider_template_name = "get_provider_info" - get_provider_file_path = os.path.abspath( - os.path.join( - get_target_providers_package_folder(provider_package_id), - "get_provider_info.py", - ) - ) - get_provider_content = render_template( - template_name=get_provider_template_name, - context=context, - extension=".py", - autoescape=False, - keep_trailing_newline=True, - ) - with open(get_provider_file_path, "wt") as get_provider_file: - get_provider_file.write(black_format(get_provider_content)) - - -def prepare_manifest_in_file(context): - target = os.path.abspath(os.path.join(get_target_folder(), "MANIFEST.in")) - content = render_template( - template_name="MANIFEST", - context=context, - extension=".in", - autoescape=False, - keep_trailing_newline=True, - ) - with open(target, "wt") as fh: - fh.write(content) - - def get_all_providers() -> list[str]: """ Returns all providers for regular packages. @@ -1567,25 +1386,22 @@ def tag_exists_for_version(provider_package_id: str, current_tag: str, verbose: @argument_package_id @option_verbose @option_skip_tag_check -def generate_setup_files( +def check_package_releasable( version_suffix: str, git_update: bool, package_id: str, verbose: bool, skip_tag_check: bool ): """ - Generates setup files for the package. + Check if the package is releasable. See `list-providers-packages` subcommand for the possible PACKAGE_ID values """ provider_package_id = package_id - with with_group(f"Generate setup files for '{provider_package_id}'"): + with with_group(f"Check if '{provider_package_id}' is releasable"): if not skip_tag_check: current_tag = get_current_tag(provider_package_id, version_suffix, git_update, verbose) if tag_exists_for_version(provider_package_id, current_tag, verbose): console.print(f"[yellow]The tag {current_tag} exists. Not preparing the package.[/]") sys.exit(64) - if update_setup_files(provider_package_id, version_suffix): - console.print(f"[green]Generated regular package setup files for {provider_package_id}[/]") - else: - sys.exit(64) + console.print(f"[green]The {provider_package_id} is releasable[/]") def get_current_tag(provider_package_id: str, suffix: str, git_update: bool, verbose: bool): @@ -1598,99 +1414,6 @@ def get_current_tag(provider_package_id: str, suffix: str, git_update: bool, ver return current_tag -def cleanup_remnants(verbose: bool): - if verbose: - console.print("Cleaning remnants") - files = glob.glob("*.egg-info") - for file in files: - shutil.rmtree(file, ignore_errors=True) - files = glob.glob("build") - for file in files: - shutil.rmtree(file, ignore_errors=True) - - -def verify_setup_cfg_prepared(provider_package): - with open("setup.cfg") as f: - setup_content = f.read() - search_for = f"providers-{provider_package.replace('.','-')} for Apache Airflow" - if search_for not in setup_content: - console.print( - f"[red]The setup.py is probably prepared for another package. " - f"It does not contain [bold]{search_for}[/bold]![/]" - ) - console.print( - f"\nRun:\n\n[bold]./dev/provider_packages/prepare_provider_packages.py " - f"generate-setup-files {provider_package}[/bold]\n" - ) - raise Exception("Wrong setup!") - - -@cli.command() -@option_package_format -@option_git_update -@option_version_suffix -@argument_package_id -@option_verbose -@option_skip_tag_check -def build_provider_packages( - package_format: str, - git_update: bool, - version_suffix: str, - package_id: str, - verbose: bool, - skip_tag_check: bool, -): - """ - Builds provider package. - - See `list-providers-packages` subcommand for the possible PACKAGE_ID values - """ - - import tempfile - - # we cannot use context managers because if the directory gets deleted (which bdist_wheel does), - # the context manager will throw an exception when trying to delete it again - tmp_build_dir = tempfile.TemporaryDirectory().name - tmp_dist_dir = tempfile.TemporaryDirectory().name - try: - provider_package_id = package_id - with with_group(f"Prepare provider package for '{provider_package_id}'"): - if not skip_tag_check and (version_suffix.startswith("rc") or version_suffix == ""): - # For RC and official releases we check if the "officially released" version exists - # and skip the released if it was. This allows to skip packages that have not been - # marked for release. For "dev" suffixes, we always build all packages - released_tag = get_current_tag(provider_package_id, "", git_update, verbose) - if tag_exists_for_version(provider_package_id, released_tag, verbose): - console.print(f"[yellow]The tag {released_tag} exists. Skipping the package.[/]") - return False - console.print(f"Changing directory to {TARGET_PROVIDER_PACKAGES_PATH}") - os.chdir(TARGET_PROVIDER_PACKAGES_PATH) - cleanup_remnants(verbose) - provider_package = package_id - verify_setup_cfg_prepared(provider_package) - - console.print(f"Building provider package: {provider_package} in format {package_format}") - command: list[str] = ["python3", "setup.py", "build", "--build-temp", tmp_build_dir] - if version_suffix is not None: - command.extend(["egg_info", "--tag-build", version_suffix]) - if package_format in ["sdist", "both"]: - command.append("sdist") - if package_format in ["wheel", "both"]: - command.extend(["bdist_wheel", "--bdist-dir", tmp_dist_dir]) - console.print(f"Executing command: '{' '.join(command)}'") - try: - subprocess.check_call(args=command, stdout=subprocess.DEVNULL) - except subprocess.CalledProcessError as ex: - console.print("[red]The command returned an error %s", ex) - sys.exit(ex.returncode) - console.print( - f"[green]Prepared provider package {provider_package} in format {package_format}[/]" - ) - finally: - shutil.rmtree(tmp_build_dir, ignore_errors=True) - shutil.rmtree(tmp_dist_dir, ignore_errors=True) - - def find_insertion_index_for_version(content: list[str], version: str) -> tuple[int, bool]: """ Finds insertion index for the specified version from the .rst changelog content. diff --git a/dev/provider_packages/CHANGELOG_TEMPLATE.rst.jinja2 b/dev/provider_packages/templates/CHANGELOG_TEMPLATE.rst.jinja2 similarity index 100% rename from dev/provider_packages/CHANGELOG_TEMPLATE.rst.jinja2 rename to dev/provider_packages/templates/CHANGELOG_TEMPLATE.rst.jinja2 diff --git a/provider_packages/INSTALL b/dev/provider_packages/templates/INSTALL.txt.jinja2 similarity index 69% rename from provider_packages/INSTALL rename to dev/provider_packages/templates/INSTALL.txt.jinja2 index bb516dba0af7c..5d78122692b2e 100644 --- a/provider_packages/INSTALL +++ b/dev/provider_packages/templates/INSTALL.txt.jinja2 @@ -1,8 +1,8 @@ ## INSTALL / BUILD instructions for Apache Airflow Provider packages -NOTE! Those sources are only intended to be used to build Apache Airflow Provider packages, -not the Apache Airflow. They have been generated using the unreleased version of Apache Airflow. -(from main) +Those sources are only intended to be used to build Apache Airflow Provider package: + +{{ package_name }} The .tar.gz sdist package contains setup.py file that can be use to build the provider package using: diff --git a/provider_packages/LICENSE b/dev/provider_packages/templates/LICENSE.txt.jinja2 similarity index 100% rename from provider_packages/LICENSE rename to dev/provider_packages/templates/LICENSE.txt.jinja2 diff --git a/provider_packages/NOTICE b/dev/provider_packages/templates/NOTICE.txt.jinja2 similarity index 100% rename from provider_packages/NOTICE rename to dev/provider_packages/templates/NOTICE.txt.jinja2 diff --git a/dev/provider_packages/PROVIDER_COMMITS_TEMPLATE.rst.jinja2 b/dev/provider_packages/templates/PROVIDER_COMMITS_TEMPLATE.rst.jinja2 similarity index 100% rename from dev/provider_packages/PROVIDER_COMMITS_TEMPLATE.rst.jinja2 rename to dev/provider_packages/templates/PROVIDER_COMMITS_TEMPLATE.rst.jinja2 diff --git a/dev/provider_packages/PROVIDER_INDEX_TEMPLATE.rst.jinja2 b/dev/provider_packages/templates/PROVIDER_INDEX_TEMPLATE.rst.jinja2 similarity index 100% rename from dev/provider_packages/PROVIDER_INDEX_TEMPLATE.rst.jinja2 rename to dev/provider_packages/templates/PROVIDER_INDEX_TEMPLATE.rst.jinja2 diff --git a/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 b/dev/provider_packages/templates/PROVIDER_ISSUE_TEMPLATE.md.jinja2 similarity index 100% rename from dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 rename to dev/provider_packages/templates/PROVIDER_ISSUE_TEMPLATE.md.jinja2 diff --git a/dev/provider_packages/PROVIDER_README_TEMPLATE.rst.jinja2 b/dev/provider_packages/templates/PROVIDER_README_TEMPLATE.rst.jinja2 similarity index 100% rename from dev/provider_packages/PROVIDER_README_TEMPLATE.rst.jinja2 rename to dev/provider_packages/templates/PROVIDER_README_TEMPLATE.rst.jinja2 diff --git a/dev/provider_packages/templates/PYPROJECT_TEMPLATE.toml.jinja2 b/dev/provider_packages/templates/PYPROJECT_TEMPLATE.toml.jinja2 new file mode 100644 index 0000000000000..181b6147414bb --- /dev/null +++ b/dev/provider_packages/templates/PYPROJECT_TEMPLATE.toml.jinja2 @@ -0,0 +1,65 @@ +[build-system] +requires = ["flit_core >=3.8.0,<4"] +build-backend = "flit.buildapi" + +[project] +name = "{{ package_name }}" +version = "{{ versions[0] }}" +authors = [ + {name = "Apache Airflow PMC", email = "dev@airflow.apache.org"}, +] +maintainers = [ + {name = "Apache Airflow PMC", email = "dev@airflow.apache.org"}, +] +description = "Provider for Apache Airflow. Implements {{ package_name }} package" +readme = "README.rst" +requires-python = ">=3.7" +keywords = ["airflow", "airflow-provider", "{{ package_name }}"] +license = {file = "LICENSE.txt"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "License :: OSI Approved :: Apache Software License", +{%- for python_version in supported_python_versions %} + "Programming Language :: Python :: {{ python_version }}", +{%- endfor %} + "Topic :: System :: Monitoring", +] + +{% if dependencies %} +dependencies = [ +{%- for dependency in escaped_dependencies %} + "{{ dependency }}", +{%- endfor %} +] + +{%- endif %} +{% if optional_dependencies %} +[project.optional-dependencies] +{%- for extra in optional_dependencies %} +"{{ extra }}" = [ +{%- for dependency in optional_dependencies[extra] %} + "{{ dependency }}", +{%- endfor %} +] +{%- endfor %} + +{%- endif %} +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/{{ package_name }}/{{ versions[0] }}/" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Twitter" = "https://twitter.com/ApacheAirflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.{{ package_id }}.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.{{ package_id }}" diff --git a/dev/provider_packages/SETUP_TEMPLATE.cfg.jinja2 b/dev/provider_packages/templates/SETUP_TEMPLATE.cfg.jinja2 similarity index 100% rename from dev/provider_packages/SETUP_TEMPLATE.cfg.jinja2 rename to dev/provider_packages/templates/SETUP_TEMPLATE.cfg.jinja2 diff --git a/dev/provider_packages/SETUP_TEMPLATE.py.jinja2 b/dev/provider_packages/templates/SETUP_TEMPLATE.py.jinja2 similarity index 100% rename from dev/provider_packages/SETUP_TEMPLATE.py.jinja2 rename to dev/provider_packages/templates/SETUP_TEMPLATE.py.jinja2 diff --git a/dev/provider_packages/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2 b/dev/provider_packages/templates/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2 similarity index 100% rename from dev/provider_packages/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2 rename to dev/provider_packages/templates/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2 diff --git a/dev/provider_packages/get_provider_info_TEMPLATE.py.jinja2 b/dev/provider_packages/templates/get_provider_info_TEMPLATE.py.jinja2 similarity index 100% rename from dev/provider_packages/get_provider_info_TEMPLATE.py.jinja2 rename to dev/provider_packages/templates/get_provider_info_TEMPLATE.py.jinja2 diff --git a/provider_packages/.flake8 b/provider_packages/.flake8 deleted file mode 120000 index cb0568d647c0d..0000000000000 --- a/provider_packages/.flake8 +++ /dev/null @@ -1 +0,0 @@ -../.flake8 \ No newline at end of file diff --git a/provider_packages/.gitignore b/provider_packages/.gitignore deleted file mode 100644 index 07ae78b669ef3..0000000000000 --- a/provider_packages/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*.egg-info -setup.py -CHANGELOG.txt -README.md -README.rst -setup.cfg -/airflow diff --git a/provider_packages/dist b/provider_packages/dist deleted file mode 120000 index 56d4b041ce6a6..0000000000000 --- a/provider_packages/dist +++ /dev/null @@ -1 +0,0 @@ -../dist/ \ No newline at end of file diff --git a/provider_packages/pyproject.toml b/provider_packages/pyproject.toml deleted file mode 120000 index 1e11d78257137..0000000000000 --- a/provider_packages/pyproject.toml +++ /dev/null @@ -1 +0,0 @@ -../pyproject.toml \ No newline at end of file diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index b56b44af0aea6..f005033a2040f 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -69,9 +69,6 @@ services: - type: bind source: ../../../airflow target: /opt/airflow/airflow - - type: bind - source: ../../../provider_packages - target: /opt/airflow/provider_packages - type: bind source: ../../../dags target: /opt/airflow/dags @@ -96,6 +93,9 @@ services: - type: bind source: ../../../pyproject.toml target: /opt/airflow/pyproject.toml + - type: bind + source: ../../../providers + target: /opt/airflow/providers - type: bind source: ../../../pytest.ini target: /opt/airflow/pytest.ini diff --git a/scripts/in_container/run_prepare_provider_packages.sh b/scripts/in_container/run_prepare_provider_packages.sh index 9da0472957634..204721e18ee7e 100755 --- a/scripts/in_container/run_prepare_provider_packages.sh +++ b/scripts/in_container/run_prepare_provider_packages.sh @@ -18,24 +18,11 @@ # shellcheck source=scripts/in_container/_in_container_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" -function copy_sources() { - group_start "Copy sources" - echo "===================================================================================" - echo " Copying sources for provider packages" - echo "===================================================================================" - mkdir -pv "${AIRFLOW_SOURCES}/provider_packages/airflow/" - rsync -avz --exclude '*node_modules*' --delete \ - "${AIRFLOW_SOURCES}/airflow" \ - "${AIRFLOW_SOURCES}/provider_packages/" - group_end -} - - function build_provider_packages() { rm -rf dist/* local package_format_args=() - if [[ ${PACKAGE_FORMAT=} != "" ]]; then - package_format_args=("--package-format" "${PACKAGE_FORMAT}") + if [[ ${PACKAGE_FORMAT=} != "" && ${PACKAGE_FORMAT=} != "both" ]]; then + package_format_args=("--format" "${PACKAGE_FORMAT}") fi local prepared_packages=() @@ -60,11 +47,13 @@ function build_provider_packages() { local provider_package for provider_package in "${PROVIDER_PACKAGES[@]}" do + local provider_package_dir + provider_package_dir="${AIRFLOW_SOURCES}/providers/${provider_package//./\/}" rm -rf -- *.egg-info build/ local res set +e python3 "${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py" \ - generate-setup-files \ + check-package-releasable \ "${OPTIONAL_VERBOSE_FLAG[@]}" \ --no-git-update \ --version-suffix "${VERSION_SUFFIX_FOR_PYPI}" \ @@ -79,30 +68,20 @@ function build_provider_packages() { error_packages+=("${provider_package}") continue fi + pushd "${provider_package_dir}" || exit 1 + rm -rf dist/* set +e - package_suffix="" - if [[ -n ${VERSION_SUFFIX_FOR_PYPI} ]]; then - # only adds suffix to setup.py if version suffix for PyPI is set - package_suffix="${VERSION_SUFFIX_FOR_PYPI}" - fi - python3 "${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py" \ - build-provider-packages \ - "${OPTIONAL_VERBOSE_FLAG[@]}" \ - --no-git-update \ - --version-suffix "${package_suffix}" \ - "${package_format_args[@]}" \ - "${provider_package}" + # TODO: fix version suffix + flit build "${package_format_args[@]}" --setup-py res=$? set -e - if [[ ${res} == "64" ]]; then - skipped_packages+=("${provider_package}") - continue - fi + popd if [[ ${res} != "0" ]]; then error_packages+=("${provider_package}") echo "${COLOR_RED}Error when preparing ${provider_package} package${COLOR_RESET}" continue fi + mv "${provider_package_dir}/dist/"* /dist/ prepared_packages+=("${provider_package}") done echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}" @@ -141,7 +120,6 @@ install_supported_pip_version PROVIDER_PACKAGES=("${@}") get_providers_to_act_on "${@}" -copy_sources build_provider_packages echo diff --git a/scripts/in_container/verify_providers.py b/scripts/in_container/verify_providers.py index 87d79ee6ca123..b94772c43cf62 100755 --- a/scripts/in_container/verify_providers.py +++ b/scripts/in_container/verify_providers.py @@ -62,8 +62,8 @@ class ProviderPackageDetails(NamedTuple): provider_package_id: str full_package_name: str pypi_package_name: str - source_provider_package_path: str - documentation_provider_package_path: str + source_provider_package_path: Path + documentation_provider_package_path: Path provider_description: str versions: list[str] excluded_python_versions: list[str] diff --git a/setup.py b/setup.py index 8b8aaacb00371..1d03070153a9b 100644 --- a/setup.py +++ b/setup.py @@ -373,6 +373,7 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve "flake8-colors", "flake8-implicit-str-concat", "flaky", + "flit", "freezegun", "gitpython", "ipdb",