Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3d2a0b7552829f49477890b293207a4c
4cac13b21eee8b732a46c5a15aec7a4b
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
abd6c12fe888796a153522495b802278
bd80bf2dd63a27111b8c5745df5dfb86
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
010c3a7215ff699933b21bf4e5787fcb
5865810f5a4b12c9199267b7dd6459b3
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
e8af9b706de3926a3297a6c1114d7e5c
a27c1726f5902e5fdb501ecdee226476
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
33f340084282725cf473d84c59e2d63f
c33a2f6d00a3a8dbec8b56c1c2d88d54
16 changes: 8 additions & 8 deletions dev/breeze/tests/test_selective_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1121,7 +1121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
pytest.param(
("providers/amazon/src/airflow/providers/amazon/provider.yaml",),
{
"selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes common.ai "
"selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
"common.compat common.messaging common.sql databricks exasol ftp google http imap microsoft.azure "
"mongo mysql openlineage postgres salesforce ssh teradata",
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
Expand All @@ -1148,9 +1148,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
{
"description": "amazon...google",
"test_types": "Providers[amazon] Providers[apache.hive,cncf.kubernetes,"
"common.ai,common.compat,common.messaging,common.sql,databricks,exasol,ftp,"
"http,imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,"
"teradata] Providers[google]",
"common.compat,common.messaging,common.sql,databricks,exasol,ftp,http,imap,"
"microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] "
"Providers[google]",
}
]
),
Expand Down Expand Up @@ -1193,7 +1193,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
pytest.param(
("providers/amazon/src/airflow/providers/amazon/file.py",),
{
"selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes common.ai "
"selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
"common.compat common.messaging common.sql databricks exasol ftp google http imap microsoft.azure "
"mongo mysql openlineage postgres salesforce ssh teradata",
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
Expand All @@ -1217,9 +1217,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
{
"description": "amazon...google",
"test_types": "Providers[amazon] Providers[apache.hive,cncf.kubernetes,"
"common.ai,common.compat,common.messaging,common.sql,databricks,exasol,ftp,"
"http,imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,"
"teradata] Providers[google]",
"common.compat,common.messaging,common.sql,databricks,exasol,ftp,http,imap,"
"microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] "
"Providers[google]",
}
]
),
Expand Down
4 changes: 1 addition & 3 deletions providers/common/ai/docs/examples.rst
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,7 @@ Agents & tools
`example_agent_capabilities.py <https://github.com/apache/airflow/blob/providers-common-ai/|version|/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_agent_capabilities.py>`__).
* - :ref:`Toolsets <howto/toolsets>`
- Loading ``SKILL.md`` Agent Skills
(`example_agent_skills.py <https://github.com/apache/airflow/blob/providers-common-ai/|version|/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_agent_skills.py>`__),
allow-listed AWS API access via ``AWSToolset``
(`example_aws_toolset.py <https://github.com/apache/airflow/blob/providers-common-ai/|version|/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_aws_toolset.py>`__),
(`example_agent_skills.py <https://github.com/apache/airflow/blob/providers-common-ai/|version|/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_agent_skills.py>`__)
and exposing an Airflow toolset to a LangChain agent, the reverse bridge
(`example_langchain_toolset_bridge.py <https://github.com/apache/airflow/blob/providers-common-ai/|version|/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_langchain_toolset_bridge.py>`__).
* - :doc:`connections/mcp`
Expand Down
5 changes: 1 addition & 4 deletions providers/common/ai/docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -217,13 +217,12 @@ You can install such cross-provider dependencies when installing from PyPI. For

.. code-block:: bash

pip install apache-airflow-providers-common-ai[amazon]
pip install apache-airflow-providers-common-ai[common.sql]


============================================================================================================ ==============
Dependent package Extra
============================================================================================================ ==============
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_ ``amazon``
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
`apache-airflow-providers-git <https://airflow.apache.org/docs/apache-airflow-providers-git>`_ ``git``
============================================================================================================ ==============
Expand Down Expand Up @@ -253,14 +252,12 @@ Extra Dependencies
``avro`` ``fastavro>=1.10.0; python_version < "3.14"``, ``fastavro>=1.12.1; python_version >= "3.14"``
``parquet`` ``pyarrow>=18.0.0; python_version < '3.14'``, ``pyarrow>=22.0.0; python_version >= '3.14'``
``sql`` ``apache-airflow-providers-common-sql>=1.33.0``, ``sqlglot>=30.0.0``
``aws`` ``apache-airflow-providers-amazon>=9.0.0``
``common.sql`` ``apache-airflow-providers-common-sql>=1.33.0``
``langchain`` ``langchain>=1.0.0``
``llamaindex`` ``dataclasses-json>=0.6.7``, ``llama-index-core>=0.13.0``, ``llama-index-embeddings-openai>=0.6.0``, ``llama-index-llms-openai>=0.6.0``
``pdf`` ``pypdf>=4.0.0``
``docx`` ``python-docx>=1.0.0``
``git`` ``apache-airflow-providers-git``
``amazon`` ``apache-airflow-providers-amazon``
============== =======================================================================================================================================

Downloading official packages
Expand Down
74 changes: 4 additions & 70 deletions providers/common/ai/docs/toolsets.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,17 @@ Airflow's 350+ provider hooks already have typed methods, rich docstrings,
and managed credentials. Toolsets expose them as pydantic-ai tools so that
LLM agents can call them during multi-turn reasoning.

Four toolsets are included:
Three toolsets are included:

- :class:`~airflow.providers.common.ai.toolsets.aws.AWSToolset` — configured
AWS services toolset for agent access to AWS APIs through Airflow-managed
AWS connections.
- :class:`~airflow.providers.common.ai.toolsets.hook.HookToolset` — generic
adapter for any Airflow Hook.
- :class:`~airflow.providers.common.ai.toolsets.sql.SQLToolset` — curated
4-tool database toolset.
- :class:`~airflow.providers.common.ai.toolsets.mcp.MCPToolset` — connect to
`MCP servers <https://modelcontextprotocol.io/>`__ configured via Airflow
connections.
- :class:`~airflow.providers.common.ai.toolsets.sql.SQLToolset` — curated
4-tool database toolset.

All four implement pydantic-ai's
All three implement pydantic-ai's
`AbstractToolset <https://ai.pydantic.dev/toolsets/>`__ interface and can be
passed to any pydantic-ai ``Agent``, including via
:class:`~airflow.providers.common.ai.operators.agent.AgentOperator`.
Expand Down Expand Up @@ -271,69 +268,6 @@ Parameters
support DDL for in-memory tables; this guard blocks those by default.
- ``max_rows``: Maximum rows returned from the ``query`` tool. Default ``50``.

``AWSToolset``
--------------

Curated toolset that gives an agent allow-listed access to AWS APIs with
three tools:

.. list-table::
:header-rows: 1
:widths: 20 50

* - Tool
- Description
* - ``list_aws_operations``
- Lists the operations the toolset allows, grouped by service
* - ``describe_aws_operation``
- Returns an operation's parameter and response shapes (from the
botocore service model), so the agent can check what a call expects
before making it
* - ``call_aws``
- Executes an allowed operation and returns the response as JSON,
aggregating paginated results

Credentials, region, and session configuration come from the Airflow
connection (``aws_conn_id``) via the amazon provider — the model cannot
override them through tool arguments. Requires the ``aws`` extra::

pip install "apache-airflow-providers-common-ai[aws]"

.. exampleinclude:: /../../ai/src/airflow/providers/common/ai/example_dags/example_aws_toolset.py
:language: python
:start-after: [START howto_operator_agent_aws]
:end-before: [END howto_operator_agent_aws]

Access is deny-by-default: ``allowed_actions`` is required and supports ``*``
wildcards in the operation part only (``"athena:*"``, ``"s3:List*"``).
Operations that return credentials or decrypted secrets — ``sts:AssumeRole``,
``secretsmanager:GetSecretValue``, ``kms:Decrypt``, and similar — are never
matched by a wildcard and must be listed verbatim to be callable.

.. warning::
``allowed_actions`` bounds what the agent can ask for, not what the
credentials can do. Point ``aws_conn_id`` at a least-privilege IAM role
scoped to the same operations.

Parameters
^^^^^^^^^^

- ``aws_conn_id``: Airflow connection ID for AWS credentials. Default
``aws_default``.
- ``allowed_actions``: Operations the agent may call, in
``"<service>:<Operation>"`` form using boto3 service names and API
operation names. Required — there is no auto-discovery. Matching is case-
and underscore-insensitive, so ``"s3:list_buckets"`` equals
``"s3:ListBuckets"``. Validated against botocore's bundled service
definitions at instantiation time (local metadata only — no network, no
credentials).
- ``region_name``: AWS region for API calls. Default ``None`` — use the
region configured on the connection.
- ``max_items``: Upper bound on items aggregated from paginated operations.
Default ``1000``.
- ``max_output_bytes``: Upper bound on the serialized response returned to
the agent; larger payloads are clipped and flagged. Default ``65536``.

``LoggingToolset``
------------------

Expand Down
3 changes: 1 addition & 2 deletions providers/common/ai/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ name: Common AI
description: |
AI/LLM hooks and operators for Airflow pipelines using `pydantic-ai <https://ai.pydantic.dev/>`__.

state: not-ready
state: ready
lifecycle: incubation
source-date-epoch: 1784746595
build-system: hatchling
Expand Down Expand Up @@ -455,7 +455,6 @@ operators:
toolsets:
- integration-name: Common AI
python-modules:
- airflow.providers.common.ai.toolsets.aws
- airflow.providers.common.ai.toolsets.hook
- airflow.providers.common.ai.toolsets.sql
- airflow.providers.common.ai.toolsets.datafusion
Expand Down
9 changes: 0 additions & 9 deletions providers/common/ai/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,6 @@ dependencies = [
"apache-airflow-providers-common-sql>=1.33.0",
"sqlglot>=30.0.0",
]
# AWSToolset: allow-listed AWS API access for agents. The amazon provider
# supplies credential resolution (AwsBaseHook) and brings boto3/botocore.
"aws" = [
"apache-airflow-providers-amazon>=9.0.0",
]
"common.sql" = [
"apache-airflow-providers-common-sql>=1.33.0"
]
Expand All @@ -130,16 +125,12 @@ dependencies = [
"git" = [
"apache-airflow-providers-git"
]
"amazon" = [
"apache-airflow-providers-amazon"
]

[dependency-groups]
dev = [
"apache-airflow",
"apache-airflow-task-sdk",
"apache-airflow-devel-common",
"apache-airflow-providers-amazon",
"apache-airflow-providers-common-compat",
"apache-airflow-providers-common-sql",
"apache-airflow-providers-git",
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

from airflow.providers.common.ai.toolsets.hook import HookToolset

__all__ = ["AWSToolset", "HookToolset", "MCPToolset", "SQLToolset", "airflow_toolset_to_langchain_tools"]
__all__ = ["HookToolset", "MCPToolset", "SQLToolset", "airflow_toolset_to_langchain_tools"]


def __getattr__(name: str):
Expand All @@ -46,12 +46,4 @@ def __getattr__(name: str):

raise AirflowOptionalProviderFeatureException(e)
return MCPToolset
if name == "AWSToolset":
try:
from airflow.providers.common.ai.toolsets.aws import AWSToolset
except ImportError as e:
from airflow.providers.common.compat.sdk import AirflowOptionalProviderFeatureException

raise AirflowOptionalProviderFeatureException() from e
return AWSToolset
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
Loading