Speed up the java_sdk e2e Gradle builds with concurrency and caching

.github/workflows/airflow-e2e-tests.yml

@@ -113,11 +113,43 @@ jobs:
           use-uv: ${{ inputs.use-uv }}
           make-mnt-writeable-and-cleanup: true
         id: breeze
+      - name: "Restore Java SDK Gradle dependency cache"
+        # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips
+        # this. Without it each java_sdk run re-downloads the example bundles' full
+        # dependency tree (Spark alone is ~1GB) into a fresh GRADLE_USER_HOME.
+        # runner.arch keys the cache per architecture: .gradle/jdks holds
+        # arch-specific auto-provisioned JDKs and this job runs on amd64 and arm64.
+        if: inputs.e2e_test_mode == 'java_sdk'
+        id: gradle-cache
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            java-sdk/.gradle/caches
+            java-sdk/.gradle/wrapper
+            java-sdk/.gradle/jdks
+            !java-sdk/.gradle/caches/*.lock
+            !java-sdk/.gradle/caches/journal-1
+          key: java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }}  # yamllint disable-line rule:line-length
+          restore-keys: |
+            java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-
       - name: "Test e2e integration tests"
         run: breeze testing airflow-e2e-tests
         env:
           DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
           E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
+      - name: "Save Java SDK Gradle dependency cache"
+        # Saved even when the e2e tests fail: the Gradle warm-up is independent of
+        # test outcome, and actions/cache's post step would drop it on every red run.
+        if: always() && inputs.e2e_test_mode == 'java_sdk' && steps.gradle-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            java-sdk/.gradle/caches
+            java-sdk/.gradle/wrapper
+            java-sdk/.gradle/jdks
+            !java-sdk/.gradle/caches/*.lock
+            !java-sdk/.gradle/caches/journal-1
+          key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
       - name: Zip logs
         run: |
           cd ./airflow-e2e-tests && zip -r logs.zip logs

airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py

@@ -19,6 +19,7 @@
 import json
 import os
 import subprocess
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from pathlib import Path
 from shutil import copyfile, copytree, rmtree
@@ -278,29 +279,31 @@ def _setup_xcom_object_storage_integration(dot_env_file, tmp_dir):
 ]
 
 
-def _setup_java_sdk_integration(dot_env_file, tmp_dir):
-    """Set up the java_sdk E2E test mode.
-
-    Builds the Java example bundle via the Gradle wrapper, then builds a
-    Java-capable Airflow worker image, copies the JARs into the temp directory,
-    and writes the coordinator configuration.
+def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False):
+    """Run the Java SDK Gradle wrapper inside the pinned JDK container.
+
+    * --user keeps build outputs owned by the current user (not root).
+    * --network=host shares one loopback across concurrent builds so Gradle's
+      cross-process lock handover works: a UDP ping to the lock owner's port
+      (org.gradle.cache.internal.locklistener.FileLockCommunicator.pingOwner,
+      see https://github.com/gradle/gradle/blob/v8.14.4/platforms/core-execution/persistent-cache/src/main/java/org/gradle/cache/internal/locklistener/FileLockCommunicator.java)
+      would fail across isolated container network namespaces, as reported in
+      https://github.com/gradle/gradle/issues/851.
+    * --no-daemon avoids a background JVM that would outlive the container.
+    * GRADLE_USER_HOME persists the Gradle distribution and dependency cache
+      in java-sdk/.gradle/ so subsequent runs skip straight to compilation.
+    * HOME is set explicitly because --user runs as the host UID which has no
+      entry in the container's /etc/passwd; Docker would otherwise inherit the
+      image's HOME (/root) which the non-root process cannot write to.
+    * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes
+      there without nesting, and its contents are visible on the host.
     """
-    # * --user keeps build outputs owned by the current user (not root).
-    # * --no-daemon avoids a background JVM that would outlive the container.
-    # * GRADLE_USER_HOME persists the Gradle distribution and dependency cache
-    #   in java-sdk/.gradle/ so subsequent runs skip straight to compilation.
-    # * HOME is set explicitly because --user runs as the host UID which has no
-    #   entry in the container's /etc/passwd; Docker would otherwise inherit the
-    #   image's HOME (/root) which the non-root process cannot write to.
-    # * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes
-    #   there without nesting, and its contents are visible on the host.
-    console.print("[yellow]Publishing Java SDK artifacts to local Maven repository...")
-    JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
-    subprocess.run(
+    return subprocess.run(
         [
             "docker",
             "run",
             "--rm",
+            "--network=host",
             "--user",
             f"{os.getuid()}:{os.getgid()}",
             "-e",
@@ -312,72 +315,65 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
             "-v",
             f"{AIRFLOW_ROOT_PATH}:/repo",
             "-w",
-            "/repo/java-sdk",
+            workdir,
             "eclipse-temurin:17-jdk",
-            "./gradlew",
-            "publishToMavenLocal",
-            "-PskipSigning=true",
+            "/repo/java-sdk/gradlew",
             "--no-daemon",
+            *gradle_argv,
         ],
         check=True,
+        capture_output=capture_output,
+        text=True,
     )
-    # TODO: Make the following build steps parallel
+
+
+def _build_example_bundle(workdir):
+    """Build one example bundle, capturing output so concurrent builds don't interleave."""
+    try:
+        completed = _run_java_sdk_gradle_container(workdir, "bundle", capture_output=True)
+    except subprocess.CalledProcessError as e:
+        console.print(f"[red]Bundle build failed in {workdir}:")
+        console.print(e.stdout, e.stderr, sep="\n", markup=False, soft_wrap=True)
+        raise
+    console.print(f"[yellow]Bundle build finished in {workdir}:")
+    console.print(completed.stdout, completed.stderr, sep="\n", markup=False, soft_wrap=True)
+
+
+def _setup_java_sdk_integration(dot_env_file, tmp_dir):
+    """Set up the java_sdk E2E test mode.
+
+    Builds the Java SDK and Scala Spark example bundles via the Gradle wrapper,
+    then builds a Java-capable Airflow worker image, copies the JARs into the
+    temp directory, and writes the coordinator configuration.
+    """
+    console.print("[yellow]Publishing Java SDK artifacts to local Maven repository...")
+    JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+    _run_java_sdk_gradle_container("/repo/java-sdk", "publishToMavenLocal", "-PskipSigning=true")
+
+    # The example and scala_spark_example are independent Gradle builds that both
+    # consume the SDK artifact published above, so build them concurrently. Sharing
+    # a writable Gradle user home between concurrent builds is safe only because
+    # --network=host lets each build ping the other's lock-owner port (see the
+    # helper's docstring); publishToMavenLocal has already unpacked the shared
+    # wrapper distribution, so neither build races to fetch it.
+    #
     # The Gradle `bundle` task is a Copy that never prunes its destination, so
     # JARs from an earlier build linger. A stale dependency JAR with its own
     # Main-Class would make JavaCoordinator's Main-Class discovery ambiguous, so
     # start each bundle from an empty directory.
     rmtree(JAVA_SDK_EXAMPLE_LIBS_PATH, ignore_errors=True)
-    console.print("[yellow]Building Java SDK example bundle (eclipse-temurin:17-jdk)...")
-    subprocess.run(
-        [
-            "docker",
-            "run",
-            "--rm",
-            "--user",
-            f"{os.getuid()}:{os.getgid()}",
-            "-e",
-            "GRADLE_USER_HOME=/repo/java-sdk/.gradle",
-            "-e",
-            "HOME=/workspace-home",
-            "-v",
-            f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2",
-            "-v",
-            f"{AIRFLOW_ROOT_PATH}:/repo",
-            "-w",
-            "/repo/java-sdk/example",
-            "eclipse-temurin:17-jdk",
-            "../gradlew",
-            "bundle",
-            "--no-daemon",
-        ],
-        check=True,
-    )
     rmtree(SCALA_SPARK_EXAMPLE_LIBS_PATH, ignore_errors=True)
-    console.print("[yellow]Building Scala Spark example bundle (eclipse-temurin:17-jdk)...")
-    subprocess.run(
-        [
-            "docker",
-            "run",
-            "--rm",
-            "--user",
-            f"{os.getuid()}:{os.getgid()}",
-            "-e",
-            "GRADLE_USER_HOME=/repo/java-sdk/.gradle",
-            "-e",
-            "HOME=/workspace-home",
-            "-v",
-            f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2",
-            "-v",
-            f"{AIRFLOW_ROOT_PATH}:/repo",
-            "-w",
-            "/repo/java-sdk/scala_spark_example",
-            "eclipse-temurin:17-jdk",
-            "../gradlew",
-            "bundle",
-            "--no-daemon",
-        ],
-        check=True,
+    console.print(
+        "[yellow]Building Java SDK and Scala Spark example bundles concurrently (eclipse-temurin:17-jdk)..."
     )
+    example_bundle_workdirs = [
+        "/repo/java-sdk/example",
+        "/repo/java-sdk/scala_spark_example",
+    ]
+    with ThreadPoolExecutor(max_workers=len(example_bundle_workdirs)) as pool:
+        bundle_builds = [pool.submit(_build_example_bundle, workdir) for workdir in example_bundle_workdirs]
+        for build in bundle_builds:
+            build.result()
 
     # Copy compose override and Dockerfile into the temp directory.
     copyfile(JAVA_COMPOSE_PATH, tmp_dir / "java.yml")