From 6172292604c342c6e98e01870415fe40b20c7809 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 14 Mar 2026 18:26:01 +0100 Subject: [PATCH] [v3-1-test] Fix IDEA setup to skip .claude directory and add scripts module (#63607) - Add `.claude/` to EXCLUDED_PREFIXES so pyproject.toml discovery skips `.claude/worktrees/` directories - Add `.claude` to _CLEANUP_SKIP_DIRS so IML cleanup ignores it - Add `scripts` to STATIC_MODULES for the scripts distribution (cherry picked from commit 0e11c841f3161860439e9cc8b2baee4e6478feb9) Co-authored-by: Jarek Potiuk --- dev/ide_setup/setup_idea.py | 1166 +++++++++++++++++++++++++++++++++++ 1 file changed, 1166 insertions(+) create mode 100755 dev/ide_setup/setup_idea.py diff --git a/dev/ide_setup/setup_idea.py b/dev/ide_setup/setup_idea.py new file mode 100755 index 0000000000000..a19cbfb2bb6bb --- /dev/null +++ b/dev/ide_setup/setup_idea.py @@ -0,0 +1,1166 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "rich>=13.6.0", +# "packaging>=24.0", +# ] +# /// +from __future__ import annotations + +import argparse +import os +import platform +import re +import signal +import subprocess +import sys +import time +import uuid +import xml.etree.ElementTree as ET +from pathlib import Path + +from packaging.specifiers import SpecifierSet +from packaging.version import Version +from rich import print +from rich.prompt import Confirm + +ROOT_AIRFLOW_FOLDER_PATH = Path(__file__).parents[2] +IDEA_FOLDER_PATH = ROOT_AIRFLOW_FOLDER_PATH / ".idea" +AIRFLOW_IML_FILE = IDEA_FOLDER_PATH / "airflow.iml" +MODULES_XML_FILE = IDEA_FOLDER_PATH / "modules.xml" +MISC_XML_FILE = IDEA_FOLDER_PATH / "misc.xml" +IDEA_NAME_FILE = IDEA_FOLDER_PATH / ".name" +BREEZE_PATH = ROOT_AIRFLOW_FOLDER_PATH / "dev" / "breeze" + +STATIC_MODULES: list[str] = [ + "airflow-core", + "airflow-ctl", + "task-sdk", + "devel-common", + "dev", + "dev/breeze", + "docker-tests", + "kubernetes-tests", + "helm-tests", + "scripts", + "task-sdk-integration-tests", +] + +# Well-known module groups for --exclude. +MODULE_GROUPS: dict[str, str] = { + "providers": "providers/", + "shared": "shared/", + "dev": "dev", + "tests": "tests", +} + +source_root_module_pattern: str = '' + +# --------------------------------------------------------------------------- +# Exclude configuration +# --------------------------------------------------------------------------- + +# Directories excluded by pattern (matched recursively against directory names in all content roots). +# Derived from .gitignore entries that can appear at any directory level. +# NOTE: "dist" is intentionally NOT here — providers/fab and providers/edge3 have legitimate +# static asset dist/ directories whitelisted in .gitignore. +EXCLUDE_PATTERNS: list[str] = [ + # Python bytecode / packaging + "__pycache__", + "*.egg-info", + ".eggs", + "build", + "develop-eggs", + "eggs", + "sdist", + "wheels", + "downloads", + "pip-wheel-metadata", + # Test / coverage / lint caches + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".ruff-cache", + ".hypothesis", + ".cache", + ".tox", + "htmlcov", + # Node / frontend + "node_modules", + ".vite", + ".pnpm-store", + # Generated documentation + "_build", + "_doctree", + "_inventory_cache", + "_api", + # Virtualenvs (recursive — root .venv is also in ROOT_EXCLUDE_FOLDERS) + "venv", + # Infrastructure / IaC + ".terraform", + "target", + # IDE / editor directories + ".vscode", + ".cursor", + # Legacy / misc + ".scrapy", + ".ropeproject", + ".spyderproject", + ".webassets-cache", + ".ipynb_checkpoints", +] + +# Directories excluded by explicit path (relative to $MODULE_DIR$, i.e. the project root). +# Derived from root-anchored .gitignore entries (those starting with /). +ROOT_EXCLUDE_FOLDERS: list[str] = [ + ".build", + ".kube", + ".venv", + ".uv-cache", + "dist", + "files", + "logs", + "out", + "tmp", + "images", + "hive_scratch_dir", + "airflow-core/dist", + "airflow-core/src/airflow/ui/coverage", + "generated", + "docker-context-files", + "target-airflow", + "dev/breeze/.venv", + "dev/registry/output", + "dev/registry/logos", + "3rd-party-licenses", + "licenses", + "registry/src/_data/versions", +] + +# --------------------------------------------------------------------------- +# Python version helpers +# --------------------------------------------------------------------------- + +# All minor versions we consider. Keep the upper bound a step ahead of the +# latest CPython release so newly released interpreters are recognised. +_ALL_MINOR_VERSIONS = [f"3.{m}" for m in range(9, 16)] + + +def _read_requires_python(pyproject_path: Path) -> str: + """Return the ``requires-python`` value from *pyproject_path*.""" + text = pyproject_path.read_text() + match = re.search(r'requires-python\s*=\s*"([^"]+)"', text) + if not match: + print(f"[red]Error:[/] could not find requires-python in {pyproject_path}") + sys.exit(1) + return match.group(1) + + +def get_supported_python_versions(pyproject_path: Path) -> list[str]: + """Return the list of supported ``X.Y`` Python versions according to *pyproject_path*.""" + spec = SpecifierSet(_read_requires_python(pyproject_path)) + return [v for v in _ALL_MINOR_VERSIONS if Version(f"{v}.0") in spec] + + +# --------------------------------------------------------------------------- +# XML helpers +# --------------------------------------------------------------------------- + + +def _build_exclude_patterns_xml(indent: str = " ") -> str: + """Build XML lines for entries.""" + return "\n".join(f'{indent}' for pattern in EXCLUDE_PATTERNS) + + +def _build_exclude_folders_xml( + folders: list[str], indent: str = " ", url_prefix: str = "file://$MODULE_DIR$" +) -> str: + """Build XML lines for entries.""" + return "\n".join(f'{indent}' for folder in folders) + + +def _build_content_xml( + source_lines: str, + *, + include_root_excludes: bool, + indent: str = " ", + url: str = "file://$MODULE_DIR$", +) -> str: + """Build a complete element with sources, exclude folders, and exclude patterns.""" + parts = [f'{indent}'] + if source_lines: + parts.append(source_lines) + if include_root_excludes: + parts.append(_build_exclude_folders_xml(ROOT_EXCLUDE_FOLDERS, indent=f"{indent} ")) + parts.append(_build_exclude_patterns_xml(indent=f"{indent} ")) + parts.append(f"{indent}") + return "\n".join(parts) + + +# --- Templates --- + +_iml_common_components = """\ + + + + + + + """ + +single_module_modules_xml_template = """\ + + + + + + + +""" + +multi_module_modules_xml_template = """\ + + + + + + {MODULE_ENTRIES} + + +""" + +multi_module_entry_template = ( + '' +) + + +def _build_root_iml(sdk_name: str, source_lines: str = "") -> str: + """Build a complete root .iml file (with project-level excludes and common components).""" + content = _build_content_xml(source_lines, include_root_excludes=True, indent=" ") + return ( + '\n' + '\n' + ' \n' + f"{content}\n" + f' \n' + ' \n' + " \n" + f"{_iml_common_components}\n" + "" + ) + + +def _build_sub_module_iml(source_lines: str, *, sdk_name: str = "") -> str: + """Build a sub-module .iml file. + + When *sdk_name* is provided the module gets its own explicit Python SDK; + otherwise it inherits the project SDK. + """ + content = _build_content_xml(source_lines, include_root_excludes=False, indent=" ") + if sdk_name: + jdk_entry = f' ' + else: + jdk_entry = ' ' + return ( + '\n' + '\n' + ' \n' + f"{content}\n" + f"{jdk_entry}\n" + ' \n' + " \n" + ' \n' + ' \n" + "" + ) + + +misc_xml_template = """\ + + + +""" + +# --------------------------------------------------------------------------- +# uv sync / SDK detection +# --------------------------------------------------------------------------- + + +def run_uv_sync(project_dir: Path, label: str, *, python_version: str = ""): + """Run ``uv sync`` in *project_dir* to create / update its .venv. + + When *python_version* is given (e.g. ``"3.12"``), ``--python `` + is passed to ``uv sync`` so that the venv is created with that interpreter. + """ + cmd: list[str] = ["uv", "sync"] + if python_version: + cmd += ["--python", python_version] + version_info = f" (python {python_version})" if python_version else "" + print(f"[cyan]Running uv sync in {label}{version_info} ...[/]") + env = {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"} + result = subprocess.run(cmd, cwd=project_dir, env=env, check=False) + if result.returncode != 0: + print(f"[red]Error:[/] uv sync failed in {label}. Check the output above.") + sys.exit(1) + print(f"[green]uv sync completed in {label}.[/]\n") + + +def get_sdk_name(venv_dir: Path, *, label: str = "") -> str: + """Return an IntelliJ SDK name for the venv in *venv_dir*. + + Uses the ``uv (