diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 0729cfb9ca21d..a2097e5ddc610 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. --- - # Details: https://github.com/kaxil/boring-cyborg labelPRBasedOnFilePath: @@ -154,8 +153,6 @@ labelPRBasedOnFilePath: - airflow/example_dags/example_kubernetes_executor.py - airflow/providers/cncf/kubernetes/**/* - airflow/providers/celery/executors/celery_kubernetes_executor.py - - docs/apache-airflow/core-concepts/executor/kubernetes.rst - - docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst - docs/apache-airflow-providers-cncf-kubernetes/**/* - kubernetes_tests/**/* - tests/providers/cncf/kubernetes/**/* diff --git a/docs/apache-airflow/core-concepts/executor/celery.rst b/docs/apache-airflow-providers-celery/celery_executor.rst similarity index 97% rename from docs/apache-airflow/core-concepts/executor/celery.rst rename to docs/apache-airflow-providers-celery/celery_executor.rst index 1113bda53068a..855746202202b 100644 --- a/docs/apache-airflow/core-concepts/executor/celery.rst +++ b/docs/apache-airflow-providers-celery/celery_executor.rst @@ -15,9 +15,6 @@ specific language governing permissions and limitations under the License. - -.. _executor:CeleryExecutor: - Celery Executor =============== @@ -36,7 +33,7 @@ change your ``airflow.cfg`` to point the executor parameter to For more information about setting up a Celery broker, refer to the exhaustive `Celery documentation on the topic `_. -The configuration parameters of the Celery Executor can be found in :doc:`apache-airflow-providers-celery:configurations-ref`. +The configuration parameters of the Celery Executor can be found in :doc:`configurations-ref`. Here are a few imperative requirements for your workers: @@ -97,7 +94,7 @@ Some caveats: - Tasks can consume resources. Make sure your worker has enough resources to run ``worker_concurrency`` tasks - Queue names are limited to 256 characters, but each broker backend might have its own restrictions -See :doc:`/administration-and-deployment/modules_management` for details on how Python and Airflow manage modules. +See :doc:`apache-airflow:administration-and-deployment/modules_management` for details on how Python and Airflow manage modules. Architecture ------------ @@ -173,7 +170,7 @@ The components communicate with each other in many places Task execution process ---------------------- -.. figure:: ../../img/run_task_on_celery_executor.png +.. figure:: img/run_task_on_celery_executor.png :scale: 50 % Sequence diagram - task execution process diff --git a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst b/docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst similarity index 95% rename from docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst rename to docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst index 6416f5d2fa58a..2a72bd2e83272 100644 --- a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst +++ b/docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst @@ -16,8 +16,6 @@ under the License. -.. _executor:CeleryKubernetesExecutor: - CeleryKubernetes Executor ========================= @@ -36,7 +34,7 @@ An executor is chosen to run a task based on the task's queue. ``CeleryKubernetesExecutor`` inherits the scalability of the ``CeleryExecutor`` to handle the high load at the peak time and runtime isolation of the ``KubernetesExecutor``. -The configuration parameters of the Celery Executor can be found in :doc:`apache-airflow-providers-celery:configurations-ref`. +The configuration parameters of the Celery Executor can be found in :doc:`configurations-ref`. When to use CeleryKubernetesExecutor diff --git a/docs/apache-airflow/img/run_task_on_celery_executor.png b/docs/apache-airflow-providers-celery/img/run_task_on_celery_executor.png similarity index 100% rename from docs/apache-airflow/img/run_task_on_celery_executor.png rename to docs/apache-airflow-providers-celery/img/run_task_on_celery_executor.png diff --git a/docs/apache-airflow-providers-celery/index.rst b/docs/apache-airflow-providers-celery/index.rst index 342061d7fa234..8604edc1a180f 100644 --- a/docs/apache-airflow-providers-celery/index.rst +++ b/docs/apache-airflow-providers-celery/index.rst @@ -29,6 +29,14 @@ Changelog Security +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Executors + + CeleryExecutor details + CeleryKubernetesExecutor details + .. toctree:: :hidden: :maxdepth: 1 diff --git a/docs/apache-airflow/img/arch-diag-kubernetes.png b/docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes.png similarity index 100% rename from docs/apache-airflow/img/arch-diag-kubernetes.png rename to docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes.png diff --git a/docs/apache-airflow/img/arch-diag-kubernetes2.png b/docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes2.png similarity index 100% rename from docs/apache-airflow/img/arch-diag-kubernetes2.png rename to docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes2.png diff --git a/docs/apache-airflow/img/k8s-failed-pod.png b/docs/apache-airflow-providers-cncf-kubernetes/img/k8s-failed-pod.png similarity index 100% rename from docs/apache-airflow/img/k8s-failed-pod.png rename to docs/apache-airflow-providers-cncf-kubernetes/img/k8s-failed-pod.png diff --git a/docs/apache-airflow/img/k8s-happy-path.png b/docs/apache-airflow-providers-cncf-kubernetes/img/k8s-happy-path.png similarity index 100% rename from docs/apache-airflow/img/k8s-happy-path.png rename to docs/apache-airflow-providers-cncf-kubernetes/img/k8s-happy-path.png diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst b/docs/apache-airflow-providers-cncf-kubernetes/index.rst index d7c8586c08884..5f597c27479ae 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/index.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/index.rst @@ -29,6 +29,14 @@ Changelog Security +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Executors + + KubernetesExecutor details + LocalKubernetesExecutor details + .. toctree:: :hidden: :maxdepth: 1 diff --git a/docs/apache-airflow/core-concepts/executor/kubernetes.rst b/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst similarity index 98% rename from docs/apache-airflow/core-concepts/executor/kubernetes.rst rename to docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst index e19264c96e5e5..5662d7a0e1f26 100644 --- a/docs/apache-airflow/core-concepts/executor/kubernetes.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst @@ -16,7 +16,7 @@ under the License. -.. _executor:KubernetesExecutor: +.. _KubernetesExecutor: Kubernetes Executor =================== @@ -38,12 +38,12 @@ KubernetesExecutor requires a non-sqlite database in the backend. When a DAG submits a task, the KubernetesExecutor requests a worker pod from the Kubernetes API. The worker pod then runs the task, reports the result, and terminates. -.. image:: ../../img/arch-diag-kubernetes.png +.. image:: img/arch-diag-kubernetes.png One example of an Airflow deployment running on a distributed set of five nodes in a Kubernetes cluster is shown below. -.. image:: ../../img/arch-diag-kubernetes2.png +.. image:: img/arch-diag-kubernetes2.png Consistent with the regular Airflow architecture, the Workers need access to the DAG files to execute the tasks within those DAGs and interact with the Metadata repository. Also, configuration information specific to the Kubernetes Executor, such as the worker namespace and image information, needs to be specified in the Airflow Configuration file. @@ -56,7 +56,7 @@ Additionally, the Kubernetes Executor enables specification of additional featur .. Airflow_Worker -> Kubernetes: Pod completes with state "Succeeded" and k8s records in ETCD .. Kubernetes -> Airflow_Scheduler: Airflow scheduler reads "Succeeded" from k8s watcher thread .. @enduml -.. image:: ../../img/k8s-happy-path.png +.. image:: img/k8s-happy-path.png Configuration ------------- @@ -272,7 +272,7 @@ In the case where a worker dies before it can report its status to the backend D .. .. @enduml -.. image:: ../../img/k8s-failed-pod.png +.. image:: img/k8s-failed-pod.png A Kubernetes watcher is a thread that can subscribe to every change that occurs in Kubernetes' database. It is alerted when pods start, run, end, and fail. diff --git a/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst b/docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst similarity index 97% rename from docs/apache-airflow/core-concepts/executor/local_kubernetes.rst rename to docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst index d50c7fc8780bd..daed3d4d1f270 100644 --- a/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst @@ -16,7 +16,7 @@ under the License. -.. _executor:LocalKubernetesExecutor: +.. _LocalKubernetesExecutor: LocalKubernetes Executor ========================= diff --git a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst index fcfec977e7b45..33b77c5364555 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst @@ -35,7 +35,7 @@ you to create and run Pods on a Kubernetes cluster. - :ref:`EksPodOperator ` operator for `AWS Elastic Kubernetes Engine `__. .. note:: - The :doc:`Kubernetes executor ` is **not** required to use this operator. + The :doc:`Kubernetes executor ` is **not** required to use this operator. How does this operator work? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/apache-airflow-providers-openai/dask_executor.rst b/docs/apache-airflow-providers-openai/dask_executor.rst new file mode 100644 index 0000000000000..34aca5a80d223 --- /dev/null +++ b/docs/apache-airflow-providers-openai/dask_executor.rst @@ -0,0 +1,53 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Dask Executor +============= + +:class:`airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor` allows you to run Airflow tasks in a Dask Distributed cluster. + +Dask clusters can be run on a single machine or on remote networks. For complete +details, consult the `Distributed documentation `_. + +To create a cluster, first start a Scheduler: + +.. code-block:: bash + + # default settings for a local cluster + DASK_HOST=127.0.0.1 + DASK_PORT=8786 + + dask-scheduler --host $DASK_HOST --port $DASK_PORT + +Next start at least one Worker on any machine that can connect to the host: + +.. code-block:: bash + + dask-worker $DASK_HOST:$DASK_PORT + +Edit your ``airflow.cfg`` to set your executor to :class:`airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor` and provide +the Dask Scheduler address in the ``[dask]`` section. For more information on setting the configuration, +see :doc:`apache-airflow:howto/set-config`. + +Please note: + +- Each Dask worker must be able to import Airflow and any dependencies you + require. +- The DaskExecutor implements queues using + `Dask Worker Resources `_ functionality. To enable the use of + queues, start your Dask workers with resources of the same name as the desired queues and a limit of ``inf``. + E.g. ``dask-worker --resources="QUEUE1=inf,QUEUE2=inf"``. diff --git a/docs/apache-airflow/administration-and-deployment/kubernetes.rst b/docs/apache-airflow/administration-and-deployment/kubernetes.rst index 160694fa11466..77bf719e27969 100644 --- a/docs/apache-airflow/administration-and-deployment/kubernetes.rst +++ b/docs/apache-airflow/administration-and-deployment/kubernetes.rst @@ -32,7 +32,7 @@ We maintain an :doc:`official Helm chart ` for Airflow that he Kubernetes Executor ^^^^^^^^^^^^^^^^^^^ -The :doc:`Kubernetes Executor ` allows you to run all the Airflow tasks on +The :doc:`Kubernetes Executor ` allows you to run all the Airflow tasks on Kubernetes as separate Pods. KubernetesPodOperator diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst index 78cf0f37254c8..773c65316d316 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst @@ -135,7 +135,7 @@ HTTP monitoring for Celery Cluster You can optionally use Flower to monitor the health of the Celery cluster. It also provides an HTTP API that you can use to build a health check for your environment. -For details about installation, see: :ref:`executor:CeleryExecutor`. For details about usage, see: `The Flower project documentation `__. +For details about installation, see: :doc:`CeleryExecutor `. For details about usage, see: `The Flower project documentation `__. CLI Check for Celery Workers ---------------------------- diff --git a/docs/apache-airflow/administration-and-deployment/production-deployment.rst b/docs/apache-airflow/administration-and-deployment/production-deployment.rst index 281dc7f658cb2..fdbe63a048cc5 100644 --- a/docs/apache-airflow/administration-and-deployment/production-deployment.rst +++ b/docs/apache-airflow/administration-and-deployment/production-deployment.rst @@ -60,8 +60,8 @@ Airflow uses :class:`~airflow.executors.sequential_executor.SequentialExecutor` nature, the user is limited to executing at most one task at a time. ``Sequential Executor`` also pauses the scheduler when it runs a task, hence it is not recommended in a production setup. You should use the :class:`~airflow.executors.local_executor.LocalExecutor` for a single machine. -For a multi-node setup, you should use the :doc:`Kubernetes executor <../core-concepts/executor/kubernetes>` or -the :doc:`Celery executor <../core-concepts/executor/celery>`. +For a multi-node setup, you should use the :doc:`Kubernetes executor ` or +the :doc:`Celery executor `. Once you have configured the executor, it is necessary to make sure that every node in the cluster contains diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst index e166c1024797c..3fa90120eeb11 100644 --- a/docs/apache-airflow/best-practices.rst +++ b/docs/apache-airflow/best-practices.rst @@ -82,7 +82,7 @@ it difficult to check the logs of that Task from the Webserver. If that is not d Communication -------------- -Airflow executes tasks of a DAG on different servers in case you are using :doc:`Kubernetes executor ` or :doc:`Celery executor `. +Airflow executes tasks of a DAG on different servers in case you are using :doc:`KubernetesExecutor ` or :doc:`Celery executor `. Therefore, you should not store any file or config in the local filesystem as the next task is likely to run on a different server without access to it — for example, a task that downloads the data file that the next task processes. In the case of :class:`Local executor `, storing a file on disk can make retries harder e.g., your task requires a config file that is deleted by another task in DAG. diff --git a/docs/apache-airflow/core-concepts/executor/debug.rst b/docs/apache-airflow/core-concepts/debug.rst similarity index 96% rename from docs/apache-airflow/core-concepts/executor/debug.rst rename to docs/apache-airflow/core-concepts/debug.rst index 896f79ce1ca76..6dcdfa51a40d2 100644 --- a/docs/apache-airflow/core-concepts/executor/debug.rst +++ b/docs/apache-airflow/core-concepts/debug.rst @@ -15,44 +15,10 @@ specific language governing permissions and limitations under the License. -.. _executor:DebugExecutor: - -Debug Executor (deprecated) -=========================== +.. _concepts:debugging: -The :class:`~airflow.executors.debug_executor.DebugExecutor` is meant as -a debug tool and can be used from IDE. It is a single process executor that -queues :class:`~airflow.models.taskinstance.TaskInstance` and executes them by running -``_run_raw_task`` method. - -Due to its nature the executor can be used with SQLite database. When used -with sensors the executor will change sensor mode to ``reschedule`` to avoid -blocking the execution of DAG. - -Additionally ``DebugExecutor`` can be used in a fail-fast mode that will make -all other running or scheduled tasks fail immediately. To enable this option set -``AIRFLOW__DEBUG__FAIL_FAST=True`` or adjust ``fail_fast`` option in your ``airflow.cfg``. -For more information on setting the configuration, see :doc:`../../howto/set-config`. - -**IDE setup steps:** - -1. Add ``main`` block at the end of your DAG file to make it runnable. - -It will run a backfill job: - -.. code-block:: python - - if __name__ == "__main__": - from airflow.utils.state import State - - dag.clear() - dag.run() - - -2. Setup ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` in run configuration of your IDE. In - this step you should also setup all environment variables required by your DAG. - -3. Run / debug the DAG file. +Debugging Airflow DAGs +====================== Testing DAGs with dag.test() ***************************** @@ -74,7 +40,7 @@ and that's it! You can add argument such as ``execution_date`` if you want to te you can run or debug DAGs as needed. Comparison with DebugExecutor -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------- The ``dag.test`` command has the following benefits over the :class:`~airflow.executors.debug_executor.DebugExecutor` class, which is now deprecated: @@ -102,3 +68,42 @@ Run ``python -m pdb .py`` for an interactive debugging experie -> bash_command='echo 1', (Pdb) run_this_last + +.. _executor:DebugExecutor: + +Debug Executor (deprecated) +*************************** + +The :class:`~airflow.executors.debug_executor.DebugExecutor` is meant as +a debug tool and can be used from IDE. It is a single process executor that +queues :class:`~airflow.models.taskinstance.TaskInstance` and executes them by running +``_run_raw_task`` method. + +Due to its nature the executor can be used with SQLite database. When used +with sensors the executor will change sensor mode to ``reschedule`` to avoid +blocking the execution of DAG. + +Additionally ``DebugExecutor`` can be used in a fail-fast mode that will make +all other running or scheduled tasks fail immediately. To enable this option set +``AIRFLOW__DEBUG__FAIL_FAST=True`` or adjust ``fail_fast`` option in your ``airflow.cfg``. +For more information on setting the configuration, see :doc:`../../howto/set-config`. + +**IDE setup steps:** + +1. Add ``main`` block at the end of your DAG file to make it runnable. + +It will run a backfill job: + +.. code-block:: python + + if __name__ == "__main__": + from airflow.utils.state import State + + dag.clear() + dag.run() + + +2. Setup ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` in run configuration of your IDE. In + this step you should also setup all environment variables required by your DAG. + +3. Run / debug the DAG file. diff --git a/docs/apache-airflow/core-concepts/executor/index.rst b/docs/apache-airflow/core-concepts/executor/index.rst index e1cc52cf6f07c..b74fd7a7d0fa0 100644 --- a/docs/apache-airflow/core-concepts/executor/index.rst +++ b/docs/apache-airflow/core-concepts/executor/index.rst @@ -52,19 +52,19 @@ There are two types of executors - those that run tasks *locally* (inside the `` .. toctree:: :maxdepth: 1 - debug local sequential -**Remote Executors** +The DebugExecutor also exists but has been deprecated in favor of using the ``dag.test()`` command for :doc:`debugging <../debug>`. -.. toctree:: - :maxdepth: 1 - celery - celery_kubernetes - kubernetes - local_kubernetes +**Remote Executors** + +* :doc:`CeleryExecutor ` +* :doc:`CeleryKubernetesExecutor ` +* :doc:`DaskExecutor ` +* :doc:`KubernetesExecutor ` +* :doc:`KubernetesLocalExecutor ` .. note:: diff --git a/docs/apache-airflow/core-concepts/index.rst b/docs/apache-airflow/core-concepts/index.rst index 9b24d4ea3f26d..dab97ea3ff8ad 100644 --- a/docs/apache-airflow/core-concepts/index.rst +++ b/docs/apache-airflow/core-concepts/index.rst @@ -51,3 +51,10 @@ Here you can find detailed documentation about each one of the core concepts of xcoms variables params + +**Debugging** + +.. toctree:: + :maxdepth: 1 + + debug diff --git a/docs/apache-airflow/howto/docker-compose/index.rst b/docs/apache-airflow/howto/docker-compose/index.rst index 00f08915eaeaa..c17e89053af65 100644 --- a/docs/apache-airflow/howto/docker-compose/index.rst +++ b/docs/apache-airflow/howto/docker-compose/index.rst @@ -20,7 +20,7 @@ Running Airflow in Docker ######################### -This quick-start guide will allow you to quickly get Airflow up and running with :doc:`CeleryExecutor ` in Docker. +This quick-start guide will allow you to quickly get Airflow up and running with the :doc:`CeleryExecutor ` in Docker. .. caution:: This procedure can be useful for learning and exploration. However, adapting it for use in real-world situations can be complicated. Making changes to this procedure will require specialized expertise in Docker & Docker Compose, and the Airflow community may not be able to help you. @@ -99,7 +99,7 @@ Optionally, you can enable flower by adding ``--profile flower`` option, e.g. `` - ``flower`` - `The flower app `__ for monitoring the environment. It is available at ``http://localhost:5555``. -All these services allow you to run Airflow with :doc:`CeleryExecutor `. For more information, see :doc:`/core-concepts/overview`. +All these services allow you to run Airflow with the :doc:`CeleryExecutor `. For more information, see :doc:`/core-concepts/overview`. Some directories in the container are mounted, which means that their contents are synchronized between your computer and the container. diff --git a/docs/apache-airflow/img/k8s-0-worker.jpeg b/docs/apache-airflow/img/k8s-0-worker.jpeg deleted file mode 100644 index 71b1aedf612c6..0000000000000 Binary files a/docs/apache-airflow/img/k8s-0-worker.jpeg and /dev/null differ diff --git a/docs/apache-airflow/img/k8s-3-worker.jpeg b/docs/apache-airflow/img/k8s-3-worker.jpeg deleted file mode 100644 index 0d6f5ca416165..0000000000000 Binary files a/docs/apache-airflow/img/k8s-3-worker.jpeg and /dev/null differ diff --git a/docs/apache-airflow/img/k8s-5-worker.jpeg b/docs/apache-airflow/img/k8s-5-worker.jpeg deleted file mode 100644 index 30227840cf20a..0000000000000 Binary files a/docs/apache-airflow/img/k8s-5-worker.jpeg and /dev/null differ diff --git a/docs/apache-airflow/redirects.txt b/docs/apache-airflow/redirects.txt index ea2ccf3dff991..49fc7c5e6fcc1 100644 --- a/docs/apache-airflow/redirects.txt +++ b/docs/apache-airflow/redirects.txt @@ -140,6 +140,22 @@ executor/sequential.rst core-concepts/executor/sequential.rst upgrading-from-1-10/upgrade-check.rst howto/upgrading-from-1-10/upgrade-check.rst core-concepts/dag-run.rst authoring-and-scheduling/cron.rst +# Moving provider executor docs to providers +## The ``../`` indicates that it will move to the root of the docs directory, unlike the rest of +## the entries here which move to docs/apache-airflow. +## It's okay to include ``/stable/``, because there's no relationship between a version of +## Airflow and the version of any provider package. +### Kubernetes Executors +core-concepts/executor/kubernetes.rst ../apache-airflow-providers-cncf-kubernetes/stable/kubernetes_executor.html +core-concepts/executor/local_kubernetes.rst ../apache-airflow-providers-cncf-kubernetes/stable/local_kubernetes_executor.html + +### Celery Executors +core-concepts/executor/celery_kubernetes.rst ../apache-airflow-providers-celery/stable/celery_kubernetes_executor.html +core-concepts/executor/celery.rst ../apache-airflow-providers-celery/stable/celery_executor.html + +### Dask Executor +core-concepts/executor/dask.rst ../apache-airflow-providers-daskexecutor/stable/dask_executor.html + # Python API python-api-ref.rst public-airflow-interface.rst diff --git a/docs/build_docs.py b/docs/build_docs.py index 3035b9458be08..597b31512230d 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -226,6 +226,7 @@ def build_docs_for_packages( spellcheck_only: bool, jobs: int, verbose: bool, + retry: bool = False, ) -> tuple[dict[str, list[DocBuildError]], dict[str, list[SpellingError]]]: """Builds documentation for all packages and combines errors.""" all_build_errors: dict[str, list[DocBuildError]] = defaultdict(list) diff --git a/docs/exts/redirects.py b/docs/exts/redirects.py index b186da9994416..3f8d9dcb6275e 100644 --- a/docs/exts/redirects.py +++ b/docs/exts/redirects.py @@ -49,18 +49,27 @@ def generate_redirects(app): if line.startswith("#"): continue + # Split line into the original path `from_path` and where the URL should redirect to `to_path` from_path, _, to_path = line.rstrip().partition(" ") log.debug("Redirecting '%s' to '%s'", from_path, to_path) + # in_suffix is often ".rst" from_path = from_path.replace(in_suffix, ".html") to_path = to_path.replace(in_suffix, ".html") - to_path_prefix = f"..{os.path.sep}" * (len(from_path.split(os.path.sep)) - 1) + # The redirect path needs to move back to the root of the apache-airflow docs directory + # or the root of the docs directory altogether for provider packages. + if "../" and "providers" in to_path: + to_path_prefix = f"..{os.path.sep}" * (len(from_path.split(os.path.sep))) + else: + to_path_prefix = f"..{os.path.sep}" * (len(from_path.split(os.path.sep)) - 1) + to_path = to_path_prefix + to_path log.debug("Resolved redirect '%s' to '%s'", from_path, to_path) + # This will be used to save an HTML file with `TEMPLATE` formatted redirected_filename = os.path.join(app.builder.outdir, from_path) redirected_directory = os.path.dirname(redirected_filename)