From 251ac6957458a4128fd985f2dc0326ea601cfee9 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 2 Jul 2026 18:18:00 +0900 Subject: [PATCH 1/3] Build the Java SDK and Scala Spark e2e bundles concurrently The example and scala_spark_example bundles are independent Gradle builds that both consume the SDK artifact published to the local Maven repo, so the Scala Spark dependency resolution no longer has to wait for the Java example build to finish before it starts. The bundle containers run with --network=host because Gradle's cross-process lock handover pings the lock owner over loopback, which isolated container network namespaces would break, and each build's output is captured and printed under its own header so concurrent logs do not interleave. --- .../tests/airflow_e2e_tests/conftest.py | 135 ++++++++---------- 1 file changed, 63 insertions(+), 72 deletions(-) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 639d378bc4d8a..ee2effa0bcde6 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -19,6 +19,7 @@ import json import os import subprocess +from concurrent.futures import ThreadPoolExecutor from datetime import datetime from pathlib import Path from shutil import copyfile, copytree, rmtree @@ -278,29 +279,27 @@ def _setup_xcom_object_storage_integration(dot_env_file, tmp_dir): ] -def _setup_java_sdk_integration(dot_env_file, tmp_dir): - """Set up the java_sdk E2E test mode. - - Builds the Java example bundle via the Gradle wrapper, then builds a - Java-capable Airflow worker image, copies the JARs into the temp directory, - and writes the coordinator configuration. +def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False): + """Run the Java SDK Gradle wrapper inside the pinned JDK container. + + * --user keeps build outputs owned by the current user (not root). + * --network=host shares one loopback across concurrent builds so Gradle's + cross-process lock handover (a UDP ping to the lock owner) works. + * --no-daemon avoids a background JVM that would outlive the container. + * GRADLE_USER_HOME persists the Gradle distribution and dependency cache + in java-sdk/.gradle/ so subsequent runs skip straight to compilation. + * HOME is set explicitly because --user runs as the host UID which has no + entry in the container's /etc/passwd; Docker would otherwise inherit the + image's HOME (/root) which the non-root process cannot write to. + * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes + there without nesting, and its contents are visible on the host. """ - # * --user keeps build outputs owned by the current user (not root). - # * --no-daemon avoids a background JVM that would outlive the container. - # * GRADLE_USER_HOME persists the Gradle distribution and dependency cache - # in java-sdk/.gradle/ so subsequent runs skip straight to compilation. - # * HOME is set explicitly because --user runs as the host UID which has no - # entry in the container's /etc/passwd; Docker would otherwise inherit the - # image's HOME (/root) which the non-root process cannot write to. - # * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes - # there without nesting, and its contents are visible on the host. - console.print("[yellow]Publishing Java SDK artifacts to local Maven repository...") - JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True) - subprocess.run( + return subprocess.run( [ "docker", "run", "--rm", + "--network=host", "--user", f"{os.getuid()}:{os.getgid()}", "-e", @@ -312,72 +311,64 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): "-v", f"{AIRFLOW_ROOT_PATH}:/repo", "-w", - "/repo/java-sdk", + workdir, "eclipse-temurin:17-jdk", - "./gradlew", - "publishToMavenLocal", - "-PskipSigning=true", + "/repo/java-sdk/gradlew", "--no-daemon", + *gradle_argv, ], check=True, + capture_output=capture_output, + text=True, ) - # TODO: Make the following build steps parallel + + +def _build_example_bundle(workdir): + """Build one example bundle, capturing output so concurrent builds don't interleave.""" + try: + completed = _run_java_sdk_gradle_container(workdir, "bundle", capture_output=True) + except subprocess.CalledProcessError as e: + console.print(f"[red]Bundle build failed in {workdir}:") + print(e.stdout, e.stderr, sep="\n") + raise + console.print(f"[yellow]Bundle build finished in {workdir}:") + print(completed.stdout) + + +def _setup_java_sdk_integration(dot_env_file, tmp_dir): + """Set up the java_sdk E2E test mode. + + Builds the Java SDK and Scala Spark example bundles via the Gradle wrapper, + then builds a Java-capable Airflow worker image, copies the JARs into the + temp directory, and writes the coordinator configuration. + """ + console.print("[yellow]Publishing Java SDK artifacts to local Maven repository...") + JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True) + _run_java_sdk_gradle_container("/repo/java-sdk", "publishToMavenLocal", "-PskipSigning=true") + + # The example and scala_spark_example are independent Gradle builds that both + # consume the SDK artifact published above, so build them concurrently. Sharing + # a writable Gradle user home between concurrent builds is safe only because + # --network=host lets each build ping the other's lock-owner port (see the + # helper's docstring); publishToMavenLocal has already unpacked the shared + # wrapper distribution, so neither build races to fetch it. + # # The Gradle `bundle` task is a Copy that never prunes its destination, so # JARs from an earlier build linger. A stale dependency JAR with its own # Main-Class would make JavaCoordinator's Main-Class discovery ambiguous, so # start each bundle from an empty directory. rmtree(JAVA_SDK_EXAMPLE_LIBS_PATH, ignore_errors=True) - console.print("[yellow]Building Java SDK example bundle (eclipse-temurin:17-jdk)...") - subprocess.run( - [ - "docker", - "run", - "--rm", - "--user", - f"{os.getuid()}:{os.getgid()}", - "-e", - "GRADLE_USER_HOME=/repo/java-sdk/.gradle", - "-e", - "HOME=/workspace-home", - "-v", - f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2", - "-v", - f"{AIRFLOW_ROOT_PATH}:/repo", - "-w", - "/repo/java-sdk/example", - "eclipse-temurin:17-jdk", - "../gradlew", - "bundle", - "--no-daemon", - ], - check=True, - ) rmtree(SCALA_SPARK_EXAMPLE_LIBS_PATH, ignore_errors=True) - console.print("[yellow]Building Scala Spark example bundle (eclipse-temurin:17-jdk)...") - subprocess.run( - [ - "docker", - "run", - "--rm", - "--user", - f"{os.getuid()}:{os.getgid()}", - "-e", - "GRADLE_USER_HOME=/repo/java-sdk/.gradle", - "-e", - "HOME=/workspace-home", - "-v", - f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2", - "-v", - f"{AIRFLOW_ROOT_PATH}:/repo", - "-w", - "/repo/java-sdk/scala_spark_example", - "eclipse-temurin:17-jdk", - "../gradlew", - "bundle", - "--no-daemon", - ], - check=True, + console.print( + "[yellow]Building Java SDK and Scala Spark example bundles concurrently (eclipse-temurin:17-jdk)..." ) + with ThreadPoolExecutor(max_workers=2) as pool: + bundle_builds = [ + pool.submit(_build_example_bundle, "/repo/java-sdk/example"), + pool.submit(_build_example_bundle, "/repo/java-sdk/scala_spark_example"), + ] + for build in bundle_builds: + build.result() # Copy compose override and Dockerfile into the temp directory. copyfile(JAVA_COMPOSE_PATH, tmp_dir / "java.yml") From b4610d6a766fed183297201baf4e6452184c07e2 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 2 Jul 2026 18:18:43 +0900 Subject: [PATCH 2/3] Cache the Java SDK Gradle dependencies in the e2e job The java_sdk e2e job builds the SDK and its example bundles with Gradle on every canary and java-sdk-touching run, starting from an empty GRADLE_USER_HOME each time. That forces a full re-download of the example bundles' dependency tree (Spark alone is around 1GB) on every run. Reusing the resolved dependencies across runs removes that repeated download. The restore/save split saves the warm-up even when the e2e tests fail, the key hashes the Kotlin-DSL build files and version catalog that pin the SDK dependencies, and runner.arch keeps the arch-specific auto-provisioned JDKs in .gradle/jdks from crossing between amd64 and arm64 runners. --- .github/workflows/airflow-e2e-tests.yml | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 3a87ea525f7b5..05f970d6e1364 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -113,11 +113,39 @@ jobs: use-uv: ${{ inputs.use-uv }} make-mnt-writeable-and-cleanup: true id: breeze + - name: "Restore Java SDK Gradle dependency cache" + # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips + # this. Without it each java_sdk run re-downloads the example bundles' full + # dependency tree (Spark alone is ~1GB) into a fresh GRADLE_USER_HOME. + # runner.arch keys the cache per architecture: .gradle/jdks holds + # arch-specific auto-provisioned JDKs and this job runs on amd64 and arm64. + if: inputs.e2e_test_mode == 'java_sdk' + id: gradle-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + java-sdk/.gradle/caches + java-sdk/.gradle/wrapper + java-sdk/.gradle/jdks + key: java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line rule:line-length + restore-keys: | + java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}- - name: "Test e2e integration tests" run: breeze testing airflow-e2e-tests env: DOCKER_IMAGE: "${{ inputs.docker-image-tag }}" E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}" + - name: "Save Java SDK Gradle dependency cache" + # Saved even when the e2e tests fail: the Gradle warm-up is independent of + # test outcome, and actions/cache's post step would drop it on every red run. + if: always() && inputs.e2e_test_mode == 'java_sdk' && steps.gradle-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + java-sdk/.gradle/caches + java-sdk/.gradle/wrapper + java-sdk/.gradle/jdks + key: ${{ steps.gradle-cache.outputs.cache-primary-key }} - name: Zip logs run: | cd ./airflow-e2e-tests && zip -r logs.zip logs From f0287aa7c6a7937e5d19245d4267bbdc39628941 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Fri, 3 Jul 2026 03:05:12 +0000 Subject: [PATCH 3/3] Address java_sdk e2e review feedback on caching and build output Excludes churny *.lock/journal files from the Gradle cache per GitHub's own caching guidance, makes bundle build output print stdout/stderr symmetrically through Rich with markup disabled so raw Gradle text can't be misparsed as markup, ties the build pool size to the bundle list instead of a hardcoded count, and cites the Gradle lock-handover mechanism the --network=host workaround depends on. --- .github/workflows/airflow-e2e-tests.yml | 4 ++++ .../tests/airflow_e2e_tests/conftest.py | 21 ++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 05f970d6e1364..4fb1fcf8d8438 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -127,6 +127,8 @@ jobs: java-sdk/.gradle/caches java-sdk/.gradle/wrapper java-sdk/.gradle/jdks + !java-sdk/.gradle/caches/*.lock + !java-sdk/.gradle/caches/journal-1 key: java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line rule:line-length restore-keys: | java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}- @@ -145,6 +147,8 @@ jobs: java-sdk/.gradle/caches java-sdk/.gradle/wrapper java-sdk/.gradle/jdks + !java-sdk/.gradle/caches/*.lock + !java-sdk/.gradle/caches/journal-1 key: ${{ steps.gradle-cache.outputs.cache-primary-key }} - name: Zip logs run: | diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index ee2effa0bcde6..eb6ed32e9a63c 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -284,7 +284,11 @@ def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False): * --user keeps build outputs owned by the current user (not root). * --network=host shares one loopback across concurrent builds so Gradle's - cross-process lock handover (a UDP ping to the lock owner) works. + cross-process lock handover works: a UDP ping to the lock owner's port + (org.gradle.cache.internal.locklistener.FileLockCommunicator.pingOwner, + see https://github.com/gradle/gradle/blob/v8.14.4/platforms/core-execution/persistent-cache/src/main/java/org/gradle/cache/internal/locklistener/FileLockCommunicator.java) + would fail across isolated container network namespaces, as reported in + https://github.com/gradle/gradle/issues/851. * --no-daemon avoids a background JVM that would outlive the container. * GRADLE_USER_HOME persists the Gradle distribution and dependency cache in java-sdk/.gradle/ so subsequent runs skip straight to compilation. @@ -329,10 +333,10 @@ def _build_example_bundle(workdir): completed = _run_java_sdk_gradle_container(workdir, "bundle", capture_output=True) except subprocess.CalledProcessError as e: console.print(f"[red]Bundle build failed in {workdir}:") - print(e.stdout, e.stderr, sep="\n") + console.print(e.stdout, e.stderr, sep="\n", markup=False, soft_wrap=True) raise console.print(f"[yellow]Bundle build finished in {workdir}:") - print(completed.stdout) + console.print(completed.stdout, completed.stderr, sep="\n", markup=False, soft_wrap=True) def _setup_java_sdk_integration(dot_env_file, tmp_dir): @@ -362,11 +366,12 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): console.print( "[yellow]Building Java SDK and Scala Spark example bundles concurrently (eclipse-temurin:17-jdk)..." ) - with ThreadPoolExecutor(max_workers=2) as pool: - bundle_builds = [ - pool.submit(_build_example_bundle, "/repo/java-sdk/example"), - pool.submit(_build_example_bundle, "/repo/java-sdk/scala_spark_example"), - ] + example_bundle_workdirs = [ + "/repo/java-sdk/example", + "/repo/java-sdk/scala_spark_example", + ] + with ThreadPoolExecutor(max_workers=len(example_bundle_workdirs)) as pool: + bundle_builds = [pool.submit(_build_example_bundle, workdir) for workdir in example_bundle_workdirs] for build in bundle_builds: build.result()