From 7f152b0a1922870a823cb30a669929d6f3aa04a6 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 21 Jan 2026 00:50:54 +0100 Subject: [PATCH] Use mprocs by default with start-airflow This change will: * use mprocs by default for start-airflow * allows to switch to tmux by --terminal-multiplexer tmux * stores last used selection permanently for subsequent runs * allows to switch multiplexer by `breeze setup config` * only print mprocs generated config when --verbose flag is used --- Dockerfile.ci | 2 +- .../03_contributors_quick_start.rst | 31 +++++++++++++--- .../mprocs/MPROCS_QUICK_REFERENCE.md | 37 ++++++++++++++----- dev/breeze/doc/images/output-commands.svg | 2 +- dev/breeze/doc/images/output_setup_config.svg | 28 ++++++++------ dev/breeze/doc/images/output_setup_config.txt | 2 +- .../doc/images/output_start-airflow.svg | 6 +-- .../doc/images/output_start-airflow.txt | 2 +- .../airflow_breeze/commands/common_options.py | 14 ++++--- .../commands/developer_commands.py | 10 ++--- .../commands/developer_commands_config.py | 2 +- .../airflow_breeze/commands/setup_commands.py | 4 ++ .../commands/setup_commands_config.py | 1 + .../src/airflow_breeze/global_constants.py | 1 + .../src/airflow_breeze/params/shell_params.py | 5 ++- scripts/docker/entrypoint_ci.sh | 2 +- .../bin/generate_mprocs_config.py | 30 ++++++++------- 17 files changed, 117 insertions(+), 62 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 473031d18036a..93ad19afb136b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1244,7 +1244,7 @@ function environment_initialization() { fi export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation - if [[ ${USE_MPROCS:="false"} == "true" || ${USE_MPROCS} == "True" ]]; then + if [[ ${TERMINAL_MULTIPLEXER:="mprocs"} == "mprocs" ]]; then # shellcheck source=scripts/in_container/bin/run_mprocs exec run_mprocs else diff --git a/contributing-docs/03_contributors_quick_start.rst b/contributing-docs/03_contributors_quick_start.rst index 3069dce29be72..2288a4a8a4479 100644 --- a/contributing-docs/03_contributors_quick_start.rst +++ b/contributing-docs/03_contributors_quick_start.rst @@ -644,15 +644,37 @@ If ``breeze`` was started with ``breeze --python 3.10 --backend postgres`` (or s .. note:: ``stop_airflow`` is available only when ``breeze`` is started with ``breeze start-airflow``. -Using mprocs Instead of tmux +Using tmux Instead of mprocs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -By default, ``breeze start-airflow`` uses tmux to manage Airflow components. You can use mprocs as an -alternative with the ``--use-mprocs`` flag: +By default, ``breeze start-airflow`` uses mprocs to manage Airflow components. You can use ``tmux`` as an +alternative with the ``--terminal-multiplexer tmux`` argument. The last choice of yours will be remembered +for the next run, you can switch back to ``mprocs`` anytime by using ``--terminal-multiplexer mprocs``. .. code-block:: bash - breeze start-airflow --use-mprocs + breeze start-airflow --terminal-multiplexer tmux + +and + +.. code-block:: bash + + breeze start-airflow --terminal-multiplexer mprocs + +You can also switch terminal multiplexer via breeze config: + +.. code-block:: bash + + breeze setup config --terminal_multiplexer tmux + breeze setup config --terminal_multiplexer mprocs + +** Benefits of using tmux:** + +* Familiar terminal multiplexer for many developers +* More control over panes and windows +* Customizable key bindings and layouts +* Wide range of plugins and extensions +* ability to see more than one log at a time **Benefits of using mprocs:** @@ -670,7 +692,6 @@ For more information on mprocs, look at `mprocs documentation `__: * `Breeze Installation <../dev/breeze/doc/01_installation.rst>`__ diff --git a/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md b/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md index df2d2d214b5b3..63e68b9e195a5 100644 --- a/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md +++ b/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md @@ -21,24 +21,41 @@ ## Basic Command +Start airflow uses mprocs by default: + +```bash +breeze start-airflow +``` + +You can switch to using `tmux` instead by specifying the option: + ```bash -breeze start-airflow --use-mprocs +breeze start-airflow --terminal-multiplexer tmux ``` -Breeze will start Airflow components using mprocs to manage multiple processes in a single terminal window. +Once you select the terminal multiplexer, Breeze will remember your choice for future runs. + +You can also switch terminal multiplexers by running `breeze setup config`: + +```bash +breeze setup config --terminal-multiplexer mprocs +breeze setup config --terminal-multiplexer tmux +``` + + +When Breeze will start Airflow components using mprocs to manage multiple processes in a single terminal window. It will generate dynamically the required configuration based on the selected executor and options and use it, the generated configuration file is stored in a `files` folder inside the container, so that you can also use it outside of Breeze if needed. ## Common Usage Patterns -| Command | Description | -|---------------------------------------------------------------|---------------------------------| -| `breeze start-airflow --use-mprocs` | Start Airflow with mprocs | -| `breeze start-airflow --use-mprocs --debug scheduler` | Debug scheduler with mprocs | -| `breeze start-airflow --use-mprocs --executor CeleryExecutor` | Use mprocs with Celery | -| `breeze start-airflow --use-mprocs --dev-mode` | Use mprocs in dev mode | -| `breeze start-airflow --use-tmux` | Explicitly use tmux (default) | +| Command | Description | +|-----------------------------------------------------------------|-----------------------------------------------------| +| `breeze start-airflow --terminal-multiplexer mprocs` | Start Airflow with mprocs (and remember the choice) | +| `breeze start-airflow --terminal-multiplexer tmux` | Start Airflow with tmux (and remember the choice) | +| `breeze start-airflow --debug scheduler` | Debug scheduler with last selected multiplexer | +| `breeze start-airflow --dev-mode --terminal-multiplexer mprocs` | Use mprocs in dev mode (and remember the choice) | ## mprocs Keyboard Shortcuts @@ -67,7 +84,7 @@ use it outside of Breeze if needed. | Variable | Purpose | |----------------------------|---------------------------------| -| `USE_MPROCS` | Enable mprocs mode | +| `TERMNAL_MULTIPLEXER` | Use mprocs when set to "mprocs" | | `INTEGRATION_CELERY` | Enable Celery components | | `CELERY_FLOWER` | Enable Flower UI | | `STANDALONE_DAG_PROCESSOR` | Enable standalone DAG processor | diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 69d451a3037c5..e8fdd395be705 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -405,7 +405,7 @@ │--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow          Enter breeze environment and starts all Airflow components in the tmux or mprocs session.  │ +│start-airflow          Enter breeze environment and starts all Airflow components in terminal multiplexer session.│ │Compile assets if contents of www directory changed.                                       │ │build-docs             Build documents.                                                                           │ │down                   Stop running breeze environment.                                                           │ diff --git a/dev/breeze/doc/images/output_setup_config.svg b/dev/breeze/doc/images/output_setup_config.svg index b5e44dff29fff..8b814cf670c6d 100644 --- a/dev/breeze/doc/images/output_setup_config.svg +++ b/dev/breeze/doc/images/output_setup_config.svg @@ -1,4 +1,4 @@ - + │[default: sqlite]                                                              │ │--postgres-version-PVersion of Postgres used.(13 | >14< | 15 | 16 | 17 | 18)[default: 14]│ │--mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0]│ -│--auth-managerSpecify the auth manager to set(>SimpleAuthManager< | FabAuthManager)│ -│[default: SimpleAuthManager]   │ -│--cheatsheet/--no-cheatsheet-C/-cEnable/disable cheatsheet.│ -│--asciiart/--no-asciiart-A/-aEnable/disable ASCIIart.│ -│--colour/--no-colourEnable/disable Colour mode (useful for colour blind-friendly communication).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--terminal-multiplexer-tWhich terminal multiplexer to use.(>mprocs< | tmux)[default: mprocs]│ +│--auth-managerSpecify the auth manager to set(>SimpleAuthManager< | FabAuthManager)│ +│[default: SimpleAuthManager]   │ +│--cheatsheet/--no-cheatsheet-C/-cEnable/disable cheatsheet.│ +│--asciiart/--no-asciiart-A/-aEnable/disable ASCIIart.│ +│--colour/--no-colourEnable/disable Colour mode (useful for colour blind-friendly communication).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_config.txt b/dev/breeze/doc/images/output_setup_config.txt index 8b4e790ed1f30..82e368c88f232 100644 --- a/dev/breeze/doc/images/output_setup_config.txt +++ b/dev/breeze/doc/images/output_setup_config.txt @@ -1 +1 @@ -8426489bf7a3566876626ff0ee6ff7cc +74d359990f4d1c6c0a5e10aa82f07073 diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index 5798865acac57..ed17652d0641d 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -528,7 +528,7 @@ Usage:breeze start-airflow[OPTIONS] [EXTRA_ARGS]... -Enter breeze environment and starts all Airflow components in the tmux or mprocs session. Compile assets if contents  +Enter breeze environment and starts all Airflow components in terminal multiplexer session. Compile assets if contents of www directory changed. ╭─ Execution mode ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ @@ -549,8 +549,8 @@ │--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ │r(required for Airflow 3).                              │ │[default: standalone-dag-processor]                    │ -│--use-mprocs/--use-tmuxUse mprocs instead of tmux for start-airflow.│ -│[default: use-tmux]                          │ +│--terminal-multiplexer-tWhich terminal multiplexer to use.(>mprocs< | tmux)│ +│[default: mprocs]                 │ │--auth-managerSpecify the auth manager to set       │ │(>SimpleAuthManager< | FabAuthManager)│ │[default: SimpleAuthManager]          │ diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index 942b916d6db97..7fe59e444f12f 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -e59a9e6c7c9ce7b0a0bfdd8aad9e161b +b40aa6672a59264cc7cc7effa7fec31e diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py index a9f8c484314a4..d76feb7129c7c 100644 --- a/dev/breeze/src/airflow_breeze/commands/common_options.py +++ b/dev/breeze/src/airflow_breeze/commands/common_options.py @@ -31,6 +31,7 @@ ALLOWED_MYSQL_VERSIONS, ALLOWED_POSTGRES_VERSIONS, ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, + ALLOWED_TERMINAL_MULTIPLEXERS, ALLOWED_TTY, ALLOWED_USE_AIRFLOW_VERSIONS, APACHE_AIRFLOW_GITHUB_REPOSITORY, @@ -392,13 +393,14 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, help="Run standalone dag processor for start-airflow (required for Airflow 3).", envvar="STANDALONE_DAG_PROCESSOR", ) -option_use_mprocs = click.option( - "--use-mprocs/--use-tmux", - is_flag=True, - default=False, +option_terminal_multiplexer = click.option( + "-t", + "--terminal-multiplexer", + help="Which terminal multiplexer to use.", + type=CacheableChoice(ALLOWED_TERMINAL_MULTIPLEXERS), + default=CacheableDefault(ALLOWED_TERMINAL_MULTIPLEXERS[0]), show_default=True, - help="Use mprocs instead of tmux for start-airflow.", - envvar="USE_MPROCS", + envvar="TERMINAL_MULTIPLEXER", ) option_tty = click.option( "--tty", diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 4ee7c3f0de598..c6da3fd755f29 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -69,11 +69,11 @@ option_run_db_tests_only, option_skip_db_tests, option_standalone_dag_processor, + option_terminal_multiplexer, option_tty, option_upgrade_boto, option_upgrade_sqlalchemy, option_use_airflow_version, - option_use_mprocs, option_use_uv, option_uv_http_timeout, option_verbose, @@ -568,7 +568,7 @@ def shell( @option_python @option_restart @option_standalone_dag_processor -@option_use_mprocs +@option_terminal_multiplexer @option_use_uv @option_uv_http_timeout @option_use_airflow_version @@ -618,14 +618,14 @@ def start_airflow( restart: bool, skip_assets_compilation: bool, standalone_dag_processor: bool, - use_mprocs: bool, + terminal_multiplexer: str, use_airflow_version: str | None, use_distributions_from_dist: bool, use_uv: bool, uv_http_timeout: int, ): """ - Enter breeze environment and starts all Airflow components in the tmux or mprocs session. + Enter breeze environment and starts all Airflow components in terminal multiplexer session. Compile assets if contents of www directory changed. """ if dev_mode and skip_assets_compilation: @@ -712,9 +712,9 @@ def start_airflow( restart=restart, standalone_dag_processor=standalone_dag_processor, start_airflow=True, + terminal_multiplexer=terminal_multiplexer, use_airflow_version=use_airflow_version, use_distributions_from_dist=use_distributions_from_dist, - use_mprocs=use_mprocs, use_uv=use_uv, uv_http_timeout=uv_http_timeout, ) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index 10b6d7d358753..84422305cce40 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -201,7 +201,7 @@ "--platform", "--integration", "--standalone-dag-processor", - "--use-mprocs", + "--terminal-multiplexer", "--auth-manager", "--load-example-dags", "--load-default-connections", diff --git a/dev/breeze/src/airflow_breeze/commands/setup_commands.py b/dev/breeze/src/airflow_breeze/commands/setup_commands.py index 37b08dd44ec36..60d274d3d4b73 100644 --- a/dev/breeze/src/airflow_breeze/commands/setup_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/setup_commands.py @@ -39,6 +39,7 @@ option_mysql_version, option_postgres_version, option_python, + option_terminal_multiplexer, option_verbose, ) from airflow_breeze.commands.developer_commands import option_auth_manager @@ -186,6 +187,7 @@ def version(): @option_python @option_backend @option_postgres_version +@option_terminal_multiplexer @option_mysql_version @option_auth_manager @click.option("-C/-c", "--cheatsheet/--no-cheatsheet", help="Enable/disable cheatsheet.", default=None) @@ -198,6 +200,7 @@ def version(): def change_config( python: str, backend: str, + terminal_multiplexer: str, postgres_version: str, mysql_version: str, auth_manager: str, @@ -243,6 +246,7 @@ def get_suppress_status(file: str): get_console().print(f"[info]* Backend: {backend}[/]") get_console().print(f"[info]* Postgres version: {postgres_version}[/]") get_console().print(f"[info]* MySQL version: {mysql_version}[/]") + get_console().print(f"[info]* Terminal multiplexer: {terminal_multiplexer}[/]") get_console().print(f"[info]* Auth Manager: {auth_manager}[/]") get_console().print() get_console().print(f"[info]* ASCIIART: {get_suppress_status(asciiart_file)}[/]") diff --git a/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py b/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py index 685cd692409d2..9b913c1c9eca9 100644 --- a/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py @@ -63,6 +63,7 @@ "--backend", "--postgres-version", "--mysql-version", + "--terminal-multiplexer", "--auth-manager", "--cheatsheet", "--asciiart", diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index f29153d00184c..c364d4d660d3a 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -141,6 +141,7 @@ ] ) ALLOWED_TTY = ["auto", "enabled", "disabled"] +ALLOWED_TERMINAL_MULTIPLEXERS = ["mprocs", "tmux"] ALLOWED_DOCKER_COMPOSE_PROJECTS = ["breeze", "prek", "docker-compose"] ALLOWED_LOG_LEVELS = ["INFO", "DEBUG", "WARNING", "ERROR", "CRITICAL"] DEFAULT_LOG_LEVEL = ALLOWED_LOG_LEVELS[0] diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index a3e3c207049cb..ba0c4b851232f 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -35,6 +35,7 @@ ALLOWED_INSTALLATION_DISTRIBUTION_FORMATS, ALLOWED_MYSQL_VERSIONS, ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, + ALLOWED_TERMINAL_MULTIPLEXERS, APACHE_AIRFLOW_GITHUB_REPOSITORY, BREEZE_DEBUG_APISERVER_PORT, BREEZE_DEBUG_CELERY_WORKER_PORT, @@ -241,7 +242,7 @@ class ShellParams: upgrade_sqlalchemy: bool = False use_airflow_version: str | None = None use_distributions_from_dist: bool = False - use_mprocs: bool = False + terminal_multiplexer: str = ALLOWED_TERMINAL_MULTIPLEXERS[0] use_uv: bool = False use_xdist: bool = False uv_http_timeout: int = DEFAULT_UV_HTTP_TIMEOUT @@ -672,7 +673,7 @@ def env_variables_for_docker_commands(self) -> dict[str, str]: _set_var(_env, "SSH_PORT", None, SSH_PORT) _set_var(_env, "STANDALONE_DAG_PROCESSOR", self.standalone_dag_processor) _set_var(_env, "START_AIRFLOW", self.start_airflow) - _set_var(_env, "USE_MPROCS", self.use_mprocs) + _set_var(_env, "TERMINAL_MULTIPLEXER", self.terminal_multiplexer) _set_var(_env, "SUSPENDED_PROVIDERS_FOLDERS", self.suspended_providers_folders) _set_var( _env, diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 6d11039959bbf..7e291f40adbb4 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -219,7 +219,7 @@ function environment_initialization() { fi export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation - if [[ ${USE_MPROCS:="false"} == "true" || ${USE_MPROCS} == "True" ]]; then + if [[ ${TERMINAL_MULTIPLEXER:="mprocs"} == "mprocs" ]]; then # shellcheck source=scripts/in_container/bin/run_mprocs exec run_mprocs else diff --git a/scripts/in_container/bin/generate_mprocs_config.py b/scripts/in_container/bin/generate_mprocs_config.py index f793d5461ae34..5d6bbf7ffe536 100755 --- a/scripts/in_container/bin/generate_mprocs_config.py +++ b/scripts/in_container/bin/generate_mprocs_config.py @@ -202,19 +202,23 @@ def main(): with open(output_path, "w") as f: f.write(config) - # Use rich console for pretty output - console = Console() - - console.print( - f"\n[bold green]✓[/bold green] Generated mprocs configuration at: [cyan]{output_path}[/cyan]" - ) - - # Display configuration with syntax highlighting - syntax = Syntax(config, "yaml", theme="monokai", line_numbers=False) - panel = Panel( - syntax, title="[bold yellow]Configuration Preview[/bold yellow]", border_style="blue", expand=False - ) - console.print(panel) + if os.environ.get("VERBOSE", "false") == "true": + # Use rich console for pretty output + console = Console() + + console.print( + f"\n[bold green]✓[/bold green] Generated mprocs configuration at: [cyan]{output_path}[/cyan]" + ) + + # Display configuration with syntax highlighting + syntax = Syntax(config, "yaml", theme="monokai", line_numbers=False) + panel = Panel( + syntax, + title="[bold yellow]Configuration Preview[/bold yellow]", + border_style="blue", + expand=False, + ) + console.print(panel) if __name__ == "__main__":