From ad1e017b5ccb1f7149711315a29985b2a60b3191 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 8 Jul 2026 13:56:52 +0800 Subject: [PATCH 1/6] Add quickstart to Antropic provider --- providers/anthropic/docs/index.rst | 1 + providers/anthropic/docs/quickstart.rst | 119 ++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 providers/anthropic/docs/quickstart.rst diff --git a/providers/anthropic/docs/index.rst b/providers/anthropic/docs/index.rst index 582dbff0402e2..4fb3da029fdca 100644 --- a/providers/anthropic/docs/index.rst +++ b/providers/anthropic/docs/index.rst @@ -34,6 +34,7 @@ :maxdepth: 1 :caption: Guides + Quick start Connection types Operators diff --git a/providers/anthropic/docs/quickstart.rst b/providers/anthropic/docs/quickstart.rst new file mode 100644 index 0000000000000..53f5c6944555d --- /dev/null +++ b/providers/anthropic/docs/quickstart.rst @@ -0,0 +1,119 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/quickstart: + +Quick start +=========== + +Go from zero to a submitted Claude batch in three steps: install the provider, +configure a connection, and write a Dag around ``AnthropicBatchOperator``. This +provider is built around the Message Batches API and Managed Agent sessions; +for interactive, single-call LLM tasks, use ``apache-airflow-providers-common-ai`` instead. + +1. Install +---------- + +.. code-block:: bash + + pip install apache-airflow-providers-anthropic + +2. Configure the connection +--------------------------- + +Batches run through an Anthropic connection (``conn_type`` ``anthropic``, +default connection id ``anthropic_default``). For the first-party API, put +your Anthropic API key in the password field. See :ref:`howto/connection:anthropic` +for the full reference, including the ``platform`` extra for running on +Amazon Bedrock, Google Vertex AI, Claude Platform on AWS or Microsoft Foundry, +and for keyless auth via Workload Identity Federation. + +The quickest way to set one up is an environment variable: + +.. code-block:: bash + + export AIRFLOW_CONN_ANTHROPIC_DEFAULT='{"conn_type": "anthropic", "password": "sk-ant-..."}' + +Or add it through the Airflow UI (``Admin > Connections``) or the CLI (``airflow connections add``). + +3. Write your first Dag +----------------------- + +``AnthropicBatchOperator`` submits a Message Batch — a list of +``messages.create`` requests — and waits for it to reach a terminal status. +A task retry resubmits a **new** batch, so set ``retries=0``: + +.. code-block:: python + + from airflow.sdk import dag + from airflow.providers.anthropic.operators.batch import AnthropicBatchOperator + + + @dag(tags=["example"]) + def quickstart_batch(): + AnthropicBatchOperator( + task_id="submit_batch", + requests=[ + { + "custom_id": "summary-1", + "params": { + "model": "claude-opus-4-8", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "Summarize the plot of Hamlet in two sentences."} + ], + }, + }, + { + "custom_id": "summary-2", + "params": { + "model": "claude-opus-4-8", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "Summarize the plot of Macbeth in two sentences."} + ], + }, + }, + ], + wait_for_completion=True, + retries=0, + ) + + + quickstart_batch() + +Run it like any other Dag (``airflow dags test quickstart_batch``). The task +pushes the batch ID to XCom under key ``batch_id`` as soon as it submits, and +returns it once the batch reaches ``ended``. Pull the per-request results with +:meth:`~airflow.providers.anthropic.hooks.anthropic.AnthropicHook.stream_batch_results` +and write them to object storage — results can be very large and must not be +pushed to XCom. + +Where to go next +---------------- + +- :doc:`operators/anthropic` — full parameter reference for + ``AnthropicBatchOperator``, ``AnthropicBatchSensor`` (poll an + already-submitted batch without resubmitting on retry) and + ``AnthropicAgentSessionOperator`` (Anthropic-hosted Managed Agent sessions). +- :doc:`connections` — the ``platform`` extra (bedrock, vertex, aws, foundry) + and Workload Identity Federation for keyless auth. +- For interactive, single-call or agentic LLM workloads, prefer the + vendor-agnostic ``apache-airflow-providers-common-ai`` provider with + ``model="anthropic:claude-opus-4-8"``; this provider focuses on the + batch/async surface and direct SDK access that the agent abstraction does + not model. From 5e0bdbcd8c55645a5116cb1dcf0c61258562473a Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 8 Jul 2026 16:05:32 +0800 Subject: [PATCH 2/6] fixup! Add quickstart to Antropic provider --- providers/anthropic/docs/quickstart.rst | 43 ++------------ .../anthropic/example_dags/__init__.py | 0 .../anthropic/example_dags/example_batch.py | 57 +++++++++++++++++++ 3 files changed, 62 insertions(+), 38 deletions(-) create mode 100644 providers/anthropic/src/airflow/providers/anthropic/example_dags/__init__.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py diff --git a/providers/anthropic/docs/quickstart.rst b/providers/anthropic/docs/quickstart.rst index 53f5c6944555d..756d52c73022f 100644 --- a/providers/anthropic/docs/quickstart.rst +++ b/providers/anthropic/docs/quickstart.rst @@ -57,44 +57,11 @@ Or add it through the Airflow UI (``Admin > Connections``) or the CLI (``airflow ``messages.create`` requests — and waits for it to reach a terminal status. A task retry resubmits a **new** batch, so set ``retries=0``: -.. code-block:: python - - from airflow.sdk import dag - from airflow.providers.anthropic.operators.batch import AnthropicBatchOperator - - - @dag(tags=["example"]) - def quickstart_batch(): - AnthropicBatchOperator( - task_id="submit_batch", - requests=[ - { - "custom_id": "summary-1", - "params": { - "model": "claude-opus-4-8", - "max_tokens": 1024, - "messages": [ - {"role": "user", "content": "Summarize the plot of Hamlet in two sentences."} - ], - }, - }, - { - "custom_id": "summary-2", - "params": { - "model": "claude-opus-4-8", - "max_tokens": 1024, - "messages": [ - {"role": "user", "content": "Summarize the plot of Macbeth in two sentences."} - ], - }, - }, - ], - wait_for_completion=True, - retries=0, - ) - - - quickstart_batch() +.. exampleinclude:: /../src/airflow/example_dags/example_batch.py + :language: python + :start-after: [START quickstart_batch] + :end-before: [END quickstart_batch] + Run it like any other Dag (``airflow dags test quickstart_batch``). The task pushes the batch ID to XCom under key ``batch_id`` as soon as it submits, and diff --git a/providers/anthropic/src/airflow/providers/anthropic/example_dags/__init__.py b/providers/anthropic/src/airflow/providers/anthropic/example_dags/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py b/providers/anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py new file mode 100644 index 0000000000000..43f38ad3b5787 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# [START quickstart_batch] +from __future__ import annotations + +from airflow.providers.anthropic.operators.batch import AnthropicBatchOperator +from airflow.sdk import dag + + +@dag(tags=["example"]) +def quickstart_batch(): + AnthropicBatchOperator( + task_id="submit_batch", + requests=[ + { + "custom_id": "summary-1", + "params": { + "model": "claude-opus-4-8", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "Summarize the plot of Hamlet in two sentences."} + ], + }, + }, + { + "custom_id": "summary-2", + "params": { + "model": "claude-opus-4-8", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "Summarize the plot of Macbeth in two sentences."} + ], + }, + }, + ], + wait_for_completion=True, + retries=0, + ) + + +quickstart_batch() +# [end quickstart_batch] From 80b7c5c1b7c66899e5d910c149a4c16709cd5c46 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 8 Jul 2026 16:31:53 +0800 Subject: [PATCH 3/6] fixup! fixup! Add quickstart to Antropic provider --- .../providers/anthropic/example_dags/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/providers/anthropic/src/airflow/providers/anthropic/example_dags/__init__.py b/providers/anthropic/src/airflow/providers/anthropic/example_dags/__init__.py index e69de29bb2d1d..13a83393a9124 100644 --- a/providers/anthropic/src/airflow/providers/anthropic/example_dags/__init__.py +++ b/providers/anthropic/src/airflow/providers/anthropic/example_dags/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. From 32d74989c6c6a63cb5fde77d5a0665492bb4df5e Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 8 Jul 2026 16:33:20 +0800 Subject: [PATCH 4/6] fixup! fixup! fixup! Add quickstart to Antropic provider --- providers/anthropic/docs/quickstart.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/anthropic/docs/quickstart.rst b/providers/anthropic/docs/quickstart.rst index 756d52c73022f..614584523acb4 100644 --- a/providers/anthropic/docs/quickstart.rst +++ b/providers/anthropic/docs/quickstart.rst @@ -57,7 +57,8 @@ Or add it through the Airflow UI (``Admin > Connections``) or the CLI (``airflow ``messages.create`` requests — and waits for it to reach a terminal status. A task retry resubmits a **new** batch, so set ``retries=0``: -.. exampleinclude:: /../src/airflow/example_dags/example_batch.py + +.. exampleinclude:: /../../antropic/src/airflow/providers/anthropic/example_dags/example_batch.py :language: python :start-after: [START quickstart_batch] :end-before: [END quickstart_batch] From 6e8ac9155a450870552c2676035c971b6fa40d5d Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 8 Jul 2026 16:48:21 +0800 Subject: [PATCH 5/6] fixup! fixup! fixup! fixup! Add quickstart to Antropic provider --- .../airflow/providers/anthropic/example_dags/example_batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py b/providers/anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py index 43f38ad3b5787..3f3b4117a2458 100644 --- a/providers/anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py +++ b/providers/anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py @@ -54,4 +54,4 @@ def quickstart_batch(): quickstart_batch() -# [end quickstart_batch] +# [END quickstart_batch] From 7bf548109a2221078023409ff3635afaa25b95c6 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 8 Jul 2026 17:10:11 +0800 Subject: [PATCH 6/6] fixup! fixup! fixup! fixup! fixup! Add quickstart to Antropic provider --- providers/anthropic/docs/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/anthropic/docs/quickstart.rst b/providers/anthropic/docs/quickstart.rst index 614584523acb4..215621a7d1fd1 100644 --- a/providers/anthropic/docs/quickstart.rst +++ b/providers/anthropic/docs/quickstart.rst @@ -58,7 +58,7 @@ Or add it through the Airflow UI (``Admin > Connections``) or the CLI (``airflow A task retry resubmits a **new** batch, so set ``retries=0``: -.. exampleinclude:: /../../antropic/src/airflow/providers/anthropic/example_dags/example_batch.py +.. exampleinclude:: /../../anthropic/src/airflow/providers/anthropic/example_dags/example_batch.py :language: python :start-after: [START quickstart_batch] :end-before: [END quickstart_batch]