From b69728eafce4c4baebba360e049d8eadf0eedb00 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 26 Nov 2025 03:44:56 +0100 Subject: [PATCH] add mprocs support for `breeze start-airflow` command For years we used tmux as the driver for breeze's start-airflow command, however there are better alternatives. One of the best ones available now which seems to perfectly match our needs is mprocs. It's a modern alternative to tmux for managing multiple processes. It provides: - Intuitive TUI with better visual feedback - Easy process management (start, stop, restart) - Mouse and keyboard navigation - Process status indicators - Better cross-platform support For now you need to use `--use-mprocs` flag to enable it, but once we test that it works well we can make it default. --- Dockerfile.ci | 22 +- .../03_contributors_quick_start.rst | 24 +- .../20_debugging_airflow_components.rst | 22 ++ .../mprocs/MPROCS_QUICK_REFERENCE.md | 99 +++++++ contributing-docs/mprocs/README.md | 123 ++++++++ contributing-docs/mprocs/basic.yaml | 58 ++++ dev/breeze/doc/03_developer_tasks.rst | 1 + dev/breeze/doc/images/output-commands.svg | 4 +- .../doc/images/output_start-airflow.svg | 276 +++++++++--------- .../doc/images/output_start-airflow.txt | 2 +- .../airflow_breeze/commands/common_options.py | 8 + .../commands/developer_commands.py | 6 +- .../commands/developer_commands_config.py | 1 + .../src/airflow_breeze/global_constants.py | 1 + .../src/airflow_breeze/params/shell_params.py | 4 + .../src/airflow_breeze/utils/visuals.py | 2 + scripts/docker/entrypoint_ci.sh | 12 +- .../bin/generate_mprocs_config.py | 215 ++++++++++++++ scripts/in_container/bin/run_mprocs | 36 +++ scripts/in_container/bin/run_tmux | 9 - 20 files changed, 772 insertions(+), 153 deletions(-) create mode 100644 contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md create mode 100644 contributing-docs/mprocs/README.md create mode 100644 contributing-docs/mprocs/basic.yaml create mode 100755 scripts/in_container/bin/generate_mprocs_config.py create mode 100755 scripts/in_container/bin/run_mprocs diff --git a/Dockerfile.ci b/Dockerfile.ci index 22df3c67e32f6..1b4da5de15d93 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1214,10 +1214,18 @@ function environment_initialization() { fi if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then + if [[ ${BREEZE_DEBUG_CELERY_WORKER=} == "true" ]]; then + export AIRFLOW__CELERY__POOL=${AIRFLOW__CELERY__POOL:-solo} + fi export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation - # shellcheck source=scripts/in_container/bin/run_tmux - exec run_tmux + if [[ ${USE_MPROCS:="false"} == "true" || ${USE_MPROCS} == "True" ]]; then + # shellcheck source=scripts/in_container/bin/run_mprocs + exec run_mprocs + else + # shellcheck source=scripts/in_container/bin/run_tmux + exec run_tmux + fi fi } @@ -1571,6 +1579,16 @@ RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ && curl --silent --location "${HELM_URL}" | tar -xz -O "${SYSTEM}-${PLATFORM}/helm" > /usr/local/bin/helm \ && chmod +x /usr/local/bin/helm +# Install mprocs - a modern process manager for managing multiple Airflow components +ARG MPROCS_VERSION="0.7.3" + +RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ + && PLATFORM="$(uname -m)" \ + && MPROCS_URL="https://github.com/pvolok/mprocs/releases/download/v${MPROCS_VERSION}/mprocs-${MPROCS_VERSION}-${SYSTEM}-${PLATFORM}-musl.tar.gz" \ + && echo "Downloading mprocs from ${MPROCS_URL}" \ + && curl --silent --location "${MPROCS_URL}" | tar -xz -C /usr/local/bin/ mprocs \ + && chmod +x /usr/local/bin/mprocs + WORKDIR ${AIRFLOW_SOURCES} RUN mkdir -pv ${AIRFLOW_HOME} && \ diff --git a/contributing-docs/03_contributors_quick_start.rst b/contributing-docs/03_contributors_quick_start.rst index 8b7921670c4d5..ca5e85d4a0a00 100644 --- a/contributing-docs/03_contributors_quick_start.rst +++ b/contributing-docs/03_contributors_quick_start.rst @@ -641,13 +641,33 @@ If ``breeze`` was started with ``breeze --python 3.10 --backend postgres`` (or s breeze down .. note:: - ``stop_airflow`` is available only when `breeze` is started with ``breeze start-airflow``. + ``stop_airflow`` is available only when ``breeze`` is started with ``breeze start-airflow``. + +Using mprocs Instead of tmux +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ``breeze start-airflow`` uses tmux to manage Airflow components. You can use mprocs as an +alternative with the ``--use-mprocs`` flag: + +.. code-block:: bash + + breeze start-airflow --use-mprocs + +**Benefits of using mprocs:** + +* Modern terminal UI with better visual feedback +* Easier navigation with mouse and keyboard +* Individual process controls (start, stop, restart) +* Process status indicators +* Better cross-platform support + +For more information on mprocs, look at `mprocs documentation `__. 1. Knowing more about Breeze .. code-block:: bash - breeze --help + breeze --help Following are some of important topics of `Breeze documentation <../dev/breeze/doc/README.rst>`__: diff --git a/contributing-docs/20_debugging_airflow_components.rst b/contributing-docs/20_debugging_airflow_components.rst index b24b983428735..ab265a4ca2e42 100644 --- a/contributing-docs/20_debugging_airflow_components.rst +++ b/contributing-docs/20_debugging_airflow_components.rst @@ -69,6 +69,28 @@ Breeze supports two debugger options: # Use PyCharm debugger breeze start-airflow --debug scheduler --debugger pydevd-pycharm +Using mprocs Instead of tmux +----------------------------- + +By default, ``breeze start-airflow`` uses tmux to manage multiple Airflow components. You can use +mprocs as an alternative process manager with the ``--use-mprocs`` flag: + +.. code-block:: bash + + # Use mprocs instead of tmux + breeze start-airflow --use-mprocs + + # Use mprocs with debugging + breeze start-airflow --use-mprocs --debug scheduler --debug triggerer + +**Benefits of mprocs:** + +* Modern TUI with intuitive navigation +* Better keyboard shortcuts and mouse support +* Easier process management (start/stop/restart individual processes) +* Cleaner visual layout with process status indicators +* Cross-platform compatibility + Setting up VSCode for Remote Debugging -------------------------------------- diff --git a/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md b/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md new file mode 100644 index 0000000000000..25041f185d26d --- /dev/null +++ b/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md @@ -0,0 +1,99 @@ + + +# Quick Reference: mprocs Support in Breeze + +## Basic Command + +```bash +breeze start-airflow --use-mprocs +``` + +## Common Usage Patterns + +| Command | Description | +|---------------------------------------------------------------|---------------------------------| +| `breeze start-airflow --use-mprocs` | Start Airflow with mprocs | +| `breeze start-airflow --use-mprocs --debug scheduler` | Debug scheduler with mprocs | +| `breeze start-airflow --use-mprocs --executor CeleryExecutor` | Use mprocs with Celery | +| `breeze start-airflow --use-mprocs --dev-mode` | Use mprocs in dev mode | +| `breeze start-airflow --use-tmux` | Explicitly use tmux (default) | + + +## mprocs Keyboard Shortcuts + +| Key | Action | +|---------|------------------------------| +| `↑↓` | Navigate between processes | +| `Space` | Show/hide process output | +| `r` | Restart selected process | +| `k` | Kill selected process | +| `s` | Start selected process | +| `a` | Toggle showing all processes | +| `q` | Quit mprocs | +| `?` | Show help | + +## Components Managed + +- **scheduler** - Airflow scheduler +- **api_server** (3.x+) / **webserver** (2.x) - Web interface +- **triggerer** - Handles deferred tasks +- **dag_processor** - Standalone DAG processor (when enabled) +- **celery_worker** - Celery worker (with CeleryExecutor) +- **flower** - Celery monitoring (when enabled) +- **edge_worker** - Edge worker (with EdgeExecutor) + +## Environment Variables + +| Variable | Purpose | +|----------------------------|---------------------------------| +| `USE_MPROCS` | Enable mprocs mode | +| `INTEGRATION_CELERY` | Enable Celery components | +| `CELERY_FLOWER` | Enable Flower UI | +| `STANDALONE_DAG_PROCESSOR` | Enable standalone DAG processor | +| `BREEZE_DEBUG_*` | Enable component debugging | +| `DEV_MODE` | Enable development mode | + +## Debug Ports (when debugging enabled) + +| Component | Port | +|---------------|--------| +| Scheduler | 50231 | +| Dag Processor | 50232 | +| Triggerer | 50233 | +| API Server | 50234 | +| Celery Worker | 50235 | +| Edge Worker | 50236 | +| Web Server | 50237 | + +## Installation + +mprocs is **pre-installed** in the Breeze CI image as of Airflow 3.x development. No additional setup is required. + +For older images or custom setups, you can install mprocs manually: + +```bash +# Download and install mprocs +MPROCS_VERSION="0.7.3" +SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') +PLATFORM=$([ "$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64") +curl -L "https://github.com/pvolok/mprocs/releases/download/v${MPROCS_VERSION}/mprocs-${MPROCS_VERSION}-${SYSTEM}-${PLATFORM}.tar.gz" \ + | tar -xz -C /usr/local/bin/ mprocs +chmod +x /usr/local/bin/mprocs +``` diff --git a/contributing-docs/mprocs/README.md b/contributing-docs/mprocs/README.md new file mode 100644 index 0000000000000..8c0f48216997d --- /dev/null +++ b/contributing-docs/mprocs/README.md @@ -0,0 +1,123 @@ + + +# mprocs Configuration for Airflow + +This directory contains mprocs configuration files for running Airflow components. + +## Overview + +mprocs is a modern alternative to tmux for managing multiple processes. It provides: + +- Intuitive TUI with better visual feedback +- Easy process management (start, stop, restart) +- Mouse and keyboard navigation +- Process status indicators +- Better cross-platform support + +## Files + +- `basic.yaml` - Static mprocs configuration example +- `generate_mprocs_config.py` - Script to generate dynamic mprocs configuration (located in `scripts/in_container/bin/`) +- `run_mprocs` - Entry point script for starting Airflow with mprocs (located in `scripts/in_container/bin/`) + +## Usage + +### Using Breeze + +The easiest way to use mprocs with Airflow is through the Breeze command: + +```bash +breeze start-airflow --use-mprocs +``` + +This will: + +1. Start the Breeze container +2. Dynamically generate an mprocs configuration based on your environment +3. Launch mprocs with all configured Airflow components + +### With Debug Support + +You can combine mprocs with debugging: + +```bash +breeze start-airflow --use-mprocs --debug scheduler --debug triggerer +``` + +### Manual Usage + +Inside the Breeze container, you can manually generate and use mprocs configuration: + +```bash +# Generate configuration +python3 /opt/airflow/scripts/in_container/bin/generate_mprocs_config.py /tmp/airflow-mprocs.yaml + +# Start mprocs +mprocs --config /tmp/airflow-mprocs.yaml +``` + +## Dynamic Configuration + +The `generate_mprocs_config.py` script reads environment variables to determine which components to run: + +- `INTEGRATION_CELERY` - Enables Celery worker +- `CELERY_FLOWER` - Enables Flower UI +- `STANDALONE_DAG_PROCESSOR` - Enables standalone DAG processor +- `AIRFLOW__CORE__EXECUTOR` - Determines if Edge worker should run +- `USE_AIRFLOW_VERSION` - Determines if API server (3.x+) or webserver (2.x) should run +- `BREEZE_DEBUG_*` - Enables debug mode for specific components +- `DEV_MODE` - Enables development mode features + +## Components + +The following Airflow components can be managed by mprocs: + +1. **scheduler** - Main Airflow scheduler +2. **api_server** (Airflow 3.x+) or **webserver** (Airflow 2.x) +3. **triggerer** - Handles deferred tasks +4. **dag_processor** - Standalone Dag processor (optional) +5. **celery_worker** - Celery worker (when using CeleryExecutor) +6. **flower** - Celery Flower monitoring UI (optional) +7. **edge_worker** - Edge worker (when using EdgeExecutor) + +## mprocs Controls + +When mprocs is running: + +- **Arrow keys** - Navigate between processes +- **Space** - Show/hide process output +- **r** - Restart selected process +- **k** - Kill selected process +- **s** - Start selected process +- **a** - Toggle showing all processes +- **q** - Quit mprocs +- **?** - Show help + +## Requirements + +mprocs is pre-installed in the Breeze CI image starting from Airflow 3.x development. No additional setup is required. + +For older images or custom setups, you can manually install mprocs by adding it to your Dockerfile or by customizing your Breeze image. See the installation section in the Dockerfile.ci for reference. + +## See Also + +- [Debugging Airflow Components](../../contributing-docs/20_debugging_airflow_components.rst) +- [Contributors Quick Start](../../contributing-docs/03_contributors_quick_start.rst) +- [mprocs Quick Reference](../../contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md) diff --git a/contributing-docs/mprocs/basic.yaml b/contributing-docs/mprocs/basic.yaml new file mode 100644 index 0000000000000..9565fff493c4b --- /dev/null +++ b/contributing-docs/mprocs/basic.yaml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Basic mprocs configuration for Airflow +# This is a static example configuration. For dynamic configuration based on +# your environment, use the generate_mprocs_config.py script from ../in_container/bin. +--- +procs: + scheduler: + shell: airflow scheduler + restart: always + scrollback: 100000 + + api_server: + shell: airflow api-server + restart: always + scrollback: 100000 + + triggerer: + shell: airflow triggerer + restart: always + scrollback: 100000 + + dag_processor: + shell: airflow dag-processor + restart: always + scrollback: 100000 + + shell: + shell: bash + restart: always + scrollback: 100000 + + # # Optional: Uncomment to enable Celery worker (requires CeleryExecutor) + # celery_worker: + # shell: airflow celery worker + # restart: always + # scrollback: 100000 + + # # Optional: Uncomment to enable Flower (requires Celery) + # flower: + # shell: airflow celery flower + # restart: always + # scrollback: 100000 diff --git a/dev/breeze/doc/03_developer_tasks.rst b/dev/breeze/doc/03_developer_tasks.rst index dedfb1f62bdcc..7879f81aa3028 100644 --- a/dev/breeze/doc/03_developer_tasks.rst +++ b/dev/breeze/doc/03_developer_tasks.rst @@ -197,6 +197,7 @@ You can change the used host port numbers by setting appropriate environment var * ``MSSQL_HOST_PORT`` * ``FLOWER_HOST_PORT`` * ``REDIS_HOST_PORT`` +* ``RABBITMQ_HOST_PORT`` If you set these variables, next time when you enter the environment the new ports should be in effect. diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 79b74fc134f69..69d451a3037c5 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -405,8 +405,8 @@ │--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow          Enter breeze environment and starts all Airflow components in the tmux session. Compile    │ -│assets if contents of www directory changed.                                               │ +│start-airflow          Enter breeze environment and starts all Airflow components in the tmux or mprocs session.  │ +│Compile assets if contents of www directory changed.                                       │ │build-docs             Build documents.                                                                           │ │down                   Stop running breeze environment.                                                           │ │shell                  Enter breeze environment. this is the default command use when no other is selected.       │ diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index 17551b8bff590..ce6e35c4ad70e 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -1,4 +1,4 @@ - +