diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 3a87ea525f7b5..4fb1fcf8d8438 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -113,11 +113,43 @@ jobs: use-uv: ${{ inputs.use-uv }} make-mnt-writeable-and-cleanup: true id: breeze + - name: "Restore Java SDK Gradle dependency cache" + # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips + # this. Without it each java_sdk run re-downloads the example bundles' full + # dependency tree (Spark alone is ~1GB) into a fresh GRADLE_USER_HOME. + # runner.arch keys the cache per architecture: .gradle/jdks holds + # arch-specific auto-provisioned JDKs and this job runs on amd64 and arm64. + if: inputs.e2e_test_mode == 'java_sdk' + id: gradle-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + java-sdk/.gradle/caches + java-sdk/.gradle/wrapper + java-sdk/.gradle/jdks + !java-sdk/.gradle/caches/*.lock + !java-sdk/.gradle/caches/journal-1 + key: java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line rule:line-length + restore-keys: | + java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}- - name: "Test e2e integration tests" run: breeze testing airflow-e2e-tests env: DOCKER_IMAGE: "${{ inputs.docker-image-tag }}" E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}" + - name: "Save Java SDK Gradle dependency cache" + # Saved even when the e2e tests fail: the Gradle warm-up is independent of + # test outcome, and actions/cache's post step would drop it on every red run. + if: always() && inputs.e2e_test_mode == 'java_sdk' && steps.gradle-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + java-sdk/.gradle/caches + java-sdk/.gradle/wrapper + java-sdk/.gradle/jdks + !java-sdk/.gradle/caches/*.lock + !java-sdk/.gradle/caches/journal-1 + key: ${{ steps.gradle-cache.outputs.cache-primary-key }} - name: Zip logs run: | cd ./airflow-e2e-tests && zip -r logs.zip logs diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 639d378bc4d8a..eb6ed32e9a63c 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -19,6 +19,7 @@ import json import os import subprocess +from concurrent.futures import ThreadPoolExecutor from datetime import datetime from pathlib import Path from shutil import copyfile, copytree, rmtree @@ -278,29 +279,31 @@ def _setup_xcom_object_storage_integration(dot_env_file, tmp_dir): ] -def _setup_java_sdk_integration(dot_env_file, tmp_dir): - """Set up the java_sdk E2E test mode. - - Builds the Java example bundle via the Gradle wrapper, then builds a - Java-capable Airflow worker image, copies the JARs into the temp directory, - and writes the coordinator configuration. +def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False): + """Run the Java SDK Gradle wrapper inside the pinned JDK container. + + * --user keeps build outputs owned by the current user (not root). + * --network=host shares one loopback across concurrent builds so Gradle's + cross-process lock handover works: a UDP ping to the lock owner's port + (org.gradle.cache.internal.locklistener.FileLockCommunicator.pingOwner, + see https://github.com/gradle/gradle/blob/v8.14.4/platforms/core-execution/persistent-cache/src/main/java/org/gradle/cache/internal/locklistener/FileLockCommunicator.java) + would fail across isolated container network namespaces, as reported in + https://github.com/gradle/gradle/issues/851. + * --no-daemon avoids a background JVM that would outlive the container. + * GRADLE_USER_HOME persists the Gradle distribution and dependency cache + in java-sdk/.gradle/ so subsequent runs skip straight to compilation. + * HOME is set explicitly because --user runs as the host UID which has no + entry in the container's /etc/passwd; Docker would otherwise inherit the + image's HOME (/root) which the non-root process cannot write to. + * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes + there without nesting, and its contents are visible on the host. """ - # * --user keeps build outputs owned by the current user (not root). - # * --no-daemon avoids a background JVM that would outlive the container. - # * GRADLE_USER_HOME persists the Gradle distribution and dependency cache - # in java-sdk/.gradle/ so subsequent runs skip straight to compilation. - # * HOME is set explicitly because --user runs as the host UID which has no - # entry in the container's /etc/passwd; Docker would otherwise inherit the - # image's HOME (/root) which the non-root process cannot write to. - # * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes - # there without nesting, and its contents are visible on the host. - console.print("[yellow]Publishing Java SDK artifacts to local Maven repository...") - JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True) - subprocess.run( + return subprocess.run( [ "docker", "run", "--rm", + "--network=host", "--user", f"{os.getuid()}:{os.getgid()}", "-e", @@ -312,72 +315,65 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): "-v", f"{AIRFLOW_ROOT_PATH}:/repo", "-w", - "/repo/java-sdk", + workdir, "eclipse-temurin:17-jdk", - "./gradlew", - "publishToMavenLocal", - "-PskipSigning=true", + "/repo/java-sdk/gradlew", "--no-daemon", + *gradle_argv, ], check=True, + capture_output=capture_output, + text=True, ) - # TODO: Make the following build steps parallel + + +def _build_example_bundle(workdir): + """Build one example bundle, capturing output so concurrent builds don't interleave.""" + try: + completed = _run_java_sdk_gradle_container(workdir, "bundle", capture_output=True) + except subprocess.CalledProcessError as e: + console.print(f"[red]Bundle build failed in {workdir}:") + console.print(e.stdout, e.stderr, sep="\n", markup=False, soft_wrap=True) + raise + console.print(f"[yellow]Bundle build finished in {workdir}:") + console.print(completed.stdout, completed.stderr, sep="\n", markup=False, soft_wrap=True) + + +def _setup_java_sdk_integration(dot_env_file, tmp_dir): + """Set up the java_sdk E2E test mode. + + Builds the Java SDK and Scala Spark example bundles via the Gradle wrapper, + then builds a Java-capable Airflow worker image, copies the JARs into the + temp directory, and writes the coordinator configuration. + """ + console.print("[yellow]Publishing Java SDK artifacts to local Maven repository...") + JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True) + _run_java_sdk_gradle_container("/repo/java-sdk", "publishToMavenLocal", "-PskipSigning=true") + + # The example and scala_spark_example are independent Gradle builds that both + # consume the SDK artifact published above, so build them concurrently. Sharing + # a writable Gradle user home between concurrent builds is safe only because + # --network=host lets each build ping the other's lock-owner port (see the + # helper's docstring); publishToMavenLocal has already unpacked the shared + # wrapper distribution, so neither build races to fetch it. + # # The Gradle `bundle` task is a Copy that never prunes its destination, so # JARs from an earlier build linger. A stale dependency JAR with its own # Main-Class would make JavaCoordinator's Main-Class discovery ambiguous, so # start each bundle from an empty directory. rmtree(JAVA_SDK_EXAMPLE_LIBS_PATH, ignore_errors=True) - console.print("[yellow]Building Java SDK example bundle (eclipse-temurin:17-jdk)...") - subprocess.run( - [ - "docker", - "run", - "--rm", - "--user", - f"{os.getuid()}:{os.getgid()}", - "-e", - "GRADLE_USER_HOME=/repo/java-sdk/.gradle", - "-e", - "HOME=/workspace-home", - "-v", - f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2", - "-v", - f"{AIRFLOW_ROOT_PATH}:/repo", - "-w", - "/repo/java-sdk/example", - "eclipse-temurin:17-jdk", - "../gradlew", - "bundle", - "--no-daemon", - ], - check=True, - ) rmtree(SCALA_SPARK_EXAMPLE_LIBS_PATH, ignore_errors=True) - console.print("[yellow]Building Scala Spark example bundle (eclipse-temurin:17-jdk)...") - subprocess.run( - [ - "docker", - "run", - "--rm", - "--user", - f"{os.getuid()}:{os.getgid()}", - "-e", - "GRADLE_USER_HOME=/repo/java-sdk/.gradle", - "-e", - "HOME=/workspace-home", - "-v", - f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2", - "-v", - f"{AIRFLOW_ROOT_PATH}:/repo", - "-w", - "/repo/java-sdk/scala_spark_example", - "eclipse-temurin:17-jdk", - "../gradlew", - "bundle", - "--no-daemon", - ], - check=True, + console.print( + "[yellow]Building Java SDK and Scala Spark example bundles concurrently (eclipse-temurin:17-jdk)..." ) + example_bundle_workdirs = [ + "/repo/java-sdk/example", + "/repo/java-sdk/scala_spark_example", + ] + with ThreadPoolExecutor(max_workers=len(example_bundle_workdirs)) as pool: + bundle_builds = [pool.submit(_build_example_bundle, workdir) for workdir in example_bundle_workdirs] + for build in bundle_builds: + build.result() # Copy compose override and Dockerfile into the temp directory. copyfile(JAVA_COMPOSE_PATH, tmp_dir / "java.yml")