From 697897b8632e0ba872782405887bfe8a97406103 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 5 Jul 2026 12:26:55 +0200 Subject: [PATCH 1/2] Speed up lang-SDK k8s test by building Go/Java natively in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Multi-Lang KubernetesExecutor system test builds a Go bundle and a Java jar inside throwaway toolchain containers. On a dev host the module, Gradle-distribution and dependency caches persist between runs, but CI runners are ephemeral and nothing cached them, so every scheduled run re-pulled the golang/temurin images and re-downloaded the Gradle distribution and all dependencies — adding roughly ten minutes to each of the six KubernetesExecutor variants that run the test. Build the artifacts with the host toolchain in CI instead, provisioned and cached through actions/setup-go and actions/setup-java, so the toolchain image pulls and cold dependency downloads no longer happen on every run. Local runs keep the containerised build so a dev host still needs neither Go nor a JDK installed. --- .github/workflows/k8s-tests.yml | 19 ++ .../commands/kubernetes_commands.py | 241 ++++++++++-------- .../test_kubernetes_lang_sdk_commands.py | 149 +++++++++++ kubernetes-tests/lang_sdk/README.md | 7 + 4 files changed, 312 insertions(+), 104 deletions(-) create mode 100644 dev/breeze/tests/test_kubernetes_lang_sdk_commands.py diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 9aa28cfaeade8..8179f11d086d5 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -96,6 +96,24 @@ jobs: make-mnt-writeable-and-cleanup: true id: breeze # preparing k8s environment with uv takes < 15 seconds with `uv` - there is no point in caching it. + # Provision the lang-SDK Go/Java toolchains on the host (with actions/setup-* built-in caching) + # only for the single variant that runs the lang-SDK test (KubernetesExecutor, standard-naming + # off). LANG_SDK_NATIVE_TOOLCHAIN=true then makes breeze build the artifacts natively instead of + # in throwaway toolchain containers, skipping the image pulls and reusing the cached toolchains. + # keep go-version in sync with go-sdk/go.mod and the sibling Go SDK job in ci-amd.yml. + - name: "Setup Go for lang-SDK build" + if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: 1.25 + cache-dependency-path: kubernetes-tests/lang_sdk/go_example/go.sum + - name: "Setup Java for lang-SDK build" + if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 + with: + distribution: 'temurin' + java-version: '17' + cache: 'gradle' - name: "\ Run complete K8S tests ${{ matrix.executor }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-\ ${{env.KUBERNETES_VERSION}}-${{ matrix.use-standard-naming }}" @@ -106,6 +124,7 @@ jobs: # Provision + run the lang-SDK coordinator system test in a single variant only # (KubernetesExecutor, standard-naming off) to keep it off the other five k8s jobs. RUN_LANG_SDK_K8S_TESTS: ${{ (matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false) && 'true' || 'false' }} # yamllint disable-line rule:line-length + LANG_SDK_NATIVE_TOOLCHAIN: ${{ (matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false) && 'true' || 'false' }} # yamllint disable-line rule:line-length VERBOSE: "false" - name: "\ Print logs ${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-\ diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index e402f52920128..7b36aebe00b1a 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -2509,123 +2509,153 @@ def deploy_cluster( ) -def _lang_sdk_build_go_bundle(staging: Path, output: Output | None) -> None: - """Build the Go bundle into ``staging/go-artifacts`` in an ephemeral Go toolchain container. +def _lang_sdk_build_go_bundle(staging: Path, output: Output | None, *, native: bool = False) -> None: + """Build the Go bundle into ``staging/go-artifacts`` and copy the result into the staging dir. - The container means the host needs no Go install; the build writes into a gitignored dir under - the repo (with persistent caches) and the result is copied into the staging dir. + By default the build runs in an ephemeral Go toolchain container so the host needs no Go install, + writing its caches into a gitignored dir under the repo. In ``native`` mode (used in CI, where the + host already has a cached Go toolchain via ``actions/setup-go``) it invokes the host ``go`` directly, + skipping the container image pull and reusing the runner's module/build cache. """ go_dir = staging / "go-artifacts" go_dir.mkdir(parents=True, exist_ok=True) + output_bin = LANG_SDK_GO_EXAMPLE_PATH / "bin" / LANG_SDK_GO_BUNDLE_NAME - uid_gid = f"{os.getuid()}:{os.getgid()}" - go_example_ctr = f"/repo/{LANG_SDK_GO_EXAMPLE_PATH.relative_to(AIRFLOW_ROOT_PATH).as_posix()}" - - # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. - # USER/HOME must be set because the SDK calls user.Current() at init; with - # cgo disabled Go's pure-Go resolver reads those env vars and panics if - # either is empty. HOME points at a writable, gitignored dir under go_example - # so the Go build and module caches persist between runs. The package built is + # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. The package built is # the current dir (".") because go_example is its own module. - get_console(output=output).print(f"[info]Building Go bundle in {LANG_SDK_GO_BUILDER_IMAGE}") - run_command( - [ + if native: + get_console(output=output).print("[info]Building Go bundle with the host Go toolchain") + run_command( + ["go", "tool", "airflow-go-pack", "--output", str(output_bin), "."], + cwd=LANG_SDK_GO_EXAMPLE_PATH, + env={**os.environ, "CGO_ENABLED": "0"}, + output=output, + check=True, + ) + else: + uid_gid = f"{os.getuid()}:{os.getgid()}" + go_example_ctr = f"/repo/{LANG_SDK_GO_EXAMPLE_PATH.relative_to(AIRFLOW_ROOT_PATH).as_posix()}" + # USER/HOME must be set because the SDK calls user.Current() at init; with cgo disabled Go's + # pure-Go resolver reads those env vars and panics if either is empty. HOME points at a + # writable, gitignored dir under go_example so the caches persist between runs on a dev host. + get_console(output=output).print(f"[info]Building Go bundle in {LANG_SDK_GO_BUILDER_IMAGE}") + run_command( + [ + "docker", + "run", + "--rm", + "--user", + uid_gid, + "-e", + f"HOME={go_example_ctr}/.home", + "-e", + "USER=airflow", + "-e", + "CGO_ENABLED=0", + "-v", + f"{AIRFLOW_ROOT_PATH}:/repo", + "-w", + go_example_ctr, + LANG_SDK_GO_BUILDER_IMAGE, + "go", + "tool", + "airflow-go-pack", + "--output", + f"{go_example_ctr}/bin/{LANG_SDK_GO_BUNDLE_NAME}", + ".", + ], + output=output, + check=True, + ) + shutil.copy(output_bin, go_dir / LANG_SDK_GO_BUNDLE_NAME) + + +def _lang_sdk_build_java_jar(staging: Path, output: Output | None, *, native: bool = False) -> None: + """Publish the Java SDK to mavenLocal then build the java_example jar into ``staging/java-artifacts``. + + By default the build runs in an ephemeral JDK container so the host needs no JDK, persisting the + Gradle distribution/dependency and Maven caches via mounted dirs. In ``native`` mode (used in CI, + where the host already has a cached JDK + Gradle cache via ``actions/setup-java``) it invokes the + host ``./gradlew`` directly, skipping the container image pull and reusing the runner's ``~/.gradle`` + cache. ``java_example`` resolves the SDK from ``mavenLocal()``, so the SDK is published first, then + the bundle is built with java-sdk's gradle wrapper pointed at the example project (``-p``). + """ + java_dir = staging / "java-artifacts" + java_dir.mkdir(parents=True, exist_ok=True) + java_sdk_path = AIRFLOW_ROOT_PATH / "java-sdk" + + if native: + get_console(output=output).print( + "[info]Publishing Java SDK artifacts to local Maven repository with the host Gradle toolchain" + ) + run_command( + ["./gradlew", "publishToMavenLocal", "-PskipSigning=true", "--no-daemon", "--console=plain"], + cwd=java_sdk_path, + output=output, + check=True, + ) + get_console(output=output).print("[info]Building Java jar with the host Gradle toolchain") + run_command( + ["./gradlew", "-p", str(LANG_SDK_JAVA_EXAMPLE_PATH), "bundle", "--no-daemon", "--console=plain"], + cwd=java_sdk_path, + output=output, + check=True, + ) + else: + uid_gid = f"{os.getuid()}:{os.getgid()}" + java_example_ctr = f"/repo/{LANG_SDK_JAVA_EXAMPLE_PATH.relative_to(AIRFLOW_ROOT_PATH).as_posix()}" + # --user keeps build outputs owned by the host user; HOME is set explicitly because that UID has + # no /etc/passwd entry; GRADLE_USER_HOME and the mounted ~/.m2 persist the Gradle distribution and + # dependency caches between runs on a dev host. + LANG_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True) + java_docker_prefix = [ "docker", "run", "--rm", "--user", uid_gid, "-e", - f"HOME={go_example_ctr}/.home", + "GRADLE_USER_HOME=/repo/java-sdk/.gradle", "-e", - "USER=airflow", - "-e", - "CGO_ENABLED=0", + "HOME=/workspace-home", + "-v", + f"{LANG_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2", "-v", f"{AIRFLOW_ROOT_PATH}:/repo", - "-w", - go_example_ctr, - LANG_SDK_GO_BUILDER_IMAGE, - "go", - "tool", - "airflow-go-pack", - "--output", - f"{go_example_ctr}/bin/{LANG_SDK_GO_BUNDLE_NAME}", - ".", - ], - output=output, - check=True, - ) - shutil.copy(LANG_SDK_GO_EXAMPLE_PATH / "bin" / LANG_SDK_GO_BUNDLE_NAME, go_dir / LANG_SDK_GO_BUNDLE_NAME) - - -def _lang_sdk_build_java_jar(staging: Path, output: Output | None) -> None: - """Publish the Java SDK to mavenLocal then build the java_example jar into ``staging/java-artifacts``. - - Runs in an ephemeral JDK container so the host needs no JDK; the Gradle distribution, dependency - and Maven caches persist between runs via the mounted dirs. - """ - java_dir = staging / "java-artifacts" - java_dir.mkdir(parents=True, exist_ok=True) - - uid_gid = f"{os.getuid()}:{os.getgid()}" - java_example_ctr = f"/repo/{LANG_SDK_JAVA_EXAMPLE_PATH.relative_to(AIRFLOW_ROOT_PATH).as_posix()}" - - # java_example resolves the SDK plugin and libraries from mavenLocal(), so - # publish them first, then build the bundle with java-sdk's gradle wrapper - # pointed at the moved project (-p). --user keeps build outputs owned by the - # host user; HOME is set explicitly because that UID has no /etc/passwd entry; - # GRADLE_USER_HOME and the mounted ~/.m2 persist the Gradle distribution and - # dependency caches between runs. - LANG_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True) - java_docker_prefix = [ - "docker", - "run", - "--rm", - "--user", - uid_gid, - "-e", - "GRADLE_USER_HOME=/repo/java-sdk/.gradle", - "-e", - "HOME=/workspace-home", - "-v", - f"{LANG_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2", - "-v", - f"{AIRFLOW_ROOT_PATH}:/repo", - ] - get_console(output=output).print("[info]Publishing Java SDK artifacts to local Maven repository") - run_command( - [ - *java_docker_prefix, - "-w", - "/repo/java-sdk", - LANG_SDK_JAVA_BUILDER_IMAGE, - "./gradlew", - "publishToMavenLocal", - "-PskipSigning=true", - "--no-daemon", - "--console=plain", - ], - output=output, - check=True, - ) - get_console(output=output).print(f"[info]Building Java jar in {LANG_SDK_JAVA_BUILDER_IMAGE}") - run_command( - [ - *java_docker_prefix, - "-w", - "/repo/java-sdk", - LANG_SDK_JAVA_BUILDER_IMAGE, - "./gradlew", - "-p", - java_example_ctr, - "bundle", - "--no-daemon", - "--console=plain", - ], - output=output, - check=True, - ) + ] + get_console(output=output).print("[info]Publishing Java SDK artifacts to local Maven repository") + run_command( + [ + *java_docker_prefix, + "-w", + "/repo/java-sdk", + LANG_SDK_JAVA_BUILDER_IMAGE, + "./gradlew", + "publishToMavenLocal", + "-PskipSigning=true", + "--no-daemon", + "--console=plain", + ], + output=output, + check=True, + ) + get_console(output=output).print(f"[info]Building Java jar in {LANG_SDK_JAVA_BUILDER_IMAGE}") + run_command( + [ + *java_docker_prefix, + "-w", + "/repo/java-sdk", + LANG_SDK_JAVA_BUILDER_IMAGE, + "./gradlew", + "-p", + java_example_ctr, + "bundle", + "--no-daemon", + "--console=plain", + ], + output=output, + check=True, + ) jars = list((LANG_SDK_JAVA_EXAMPLE_PATH / "build" / "bundle").glob("*.jar")) if not jars: get_console(output=output).print("[error]No jar produced by the Java bundle build") @@ -2901,11 +2931,14 @@ def _setup_lang_sdk_test( # The worker-image build below produces this fixed tag; resolve it up-front so the config # rendering (which needs the tag, not the build result) does not depend on the parallel run. java_image = LANG_SDK_JAVA_WORKER_IMAGE + # In CI the Go/Java toolchains are provisioned + cached on the host (actions/setup-go, setup-java), + # so building the artifacts natively skips the toolchain-image pulls and reuses the runner caches. + native = os.environ.get("LANG_SDK_NATIVE_TOOLCHAIN", "").lower() == "true" with tempfile.TemporaryDirectory(prefix="lang_sdk_artifacts_") as tmp: staging = Path(tmp) steps: list[tuple[str, Callable[[Output | None], Any]]] = [ - ("Build Go bundle", lambda o: _lang_sdk_build_go_bundle(staging, o)), - ("Build Java jar", lambda o: _lang_sdk_build_java_jar(staging, o)), + ("Build Go bundle", lambda o: _lang_sdk_build_go_bundle(staging, o, native=native)), + ("Build Java jar", lambda o: _lang_sdk_build_java_jar(staging, o, native=native)), ("Deploy localstack", lambda o: _lang_sdk_deploy_localstack(python, kubernetes_version, o)), ] if build_java_image: diff --git a/dev/breeze/tests/test_kubernetes_lang_sdk_commands.py b/dev/breeze/tests/test_kubernetes_lang_sdk_commands.py new file mode 100644 index 0000000000000..651c0a17dc59f --- /dev/null +++ b/dev/breeze/tests/test_kubernetes_lang_sdk_commands.py @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow_breeze.commands import kubernetes_commands +from airflow_breeze.commands.kubernetes_commands import ( + _lang_sdk_build_go_bundle, + _lang_sdk_build_java_jar, +) + + +@pytest.fixture +def go_example(tmp_path, monkeypatch): + """Point the go_example dir at a tmp path (under a tmp repo root) with a pre-built bundle binary.""" + monkeypatch.setattr(kubernetes_commands, "AIRFLOW_ROOT_PATH", tmp_path) + go_dir = tmp_path / "go_example" + (go_dir / "bin").mkdir(parents=True) + (go_dir / "bin" / kubernetes_commands.LANG_SDK_GO_BUNDLE_NAME).write_text("binary") + monkeypatch.setattr(kubernetes_commands, "LANG_SDK_GO_EXAMPLE_PATH", go_dir) + return go_dir + + +@pytest.fixture +def java_example(tmp_path, monkeypatch): + """Point the java_example dir at a tmp path (under a tmp repo root) with a pre-built bundle jar.""" + monkeypatch.setattr(kubernetes_commands, "AIRFLOW_ROOT_PATH", tmp_path) + monkeypatch.setattr(kubernetes_commands, "LANG_SDK_MAVEN_CACHE_PATH", tmp_path / "m2") + java_dir = tmp_path / "java_example" + (java_dir / "build" / "bundle").mkdir(parents=True) + (java_dir / "build" / "bundle" / "app.jar").write_text("jar") + monkeypatch.setattr(kubernetes_commands, "LANG_SDK_JAVA_EXAMPLE_PATH", java_dir) + return java_dir + + +class TestLangSdkBuildGoBundle: + @mock.patch.object(kubernetes_commands, "run_command") + def test_native_uses_host_go_toolchain(self, mock_run, tmp_path, go_example): + _lang_sdk_build_go_bundle(tmp_path, None, native=True) + + cmd = mock_run.call_args.args[0] + assert cmd[:3] == ["go", "tool", "airflow-go-pack"] + assert "docker" not in cmd + assert mock_run.call_args.kwargs["cwd"] == go_example + assert mock_run.call_args.kwargs["env"]["CGO_ENABLED"] == "0" + assert (tmp_path / "go-artifacts" / kubernetes_commands.LANG_SDK_GO_BUNDLE_NAME).exists() + + @mock.patch.object(kubernetes_commands, "run_command") + def test_container_mode_runs_in_docker(self, mock_run, tmp_path, go_example): + _lang_sdk_build_go_bundle(tmp_path, None, native=False) + + cmd = mock_run.call_args.args[0] + assert cmd[0] == "docker" + assert kubernetes_commands.LANG_SDK_GO_BUILDER_IMAGE in cmd + + +class TestLangSdkBuildJavaJar: + @mock.patch.object(kubernetes_commands, "run_command") + def test_native_uses_host_gradle_toolchain(self, mock_run, tmp_path, java_example): + _lang_sdk_build_java_jar(tmp_path, None, native=True) + + publish_cmd, bundle_cmd = (call.args[0] for call in mock_run.call_args_list) + assert publish_cmd == [ + "./gradlew", + "publishToMavenLocal", + "-PskipSigning=true", + "--no-daemon", + "--console=plain", + ] + assert bundle_cmd == [ + "./gradlew", + "-p", + str(java_example), + "bundle", + "--no-daemon", + "--console=plain", + ] + assert all("docker" not in call.args[0] for call in mock_run.call_args_list) + assert (tmp_path / "java-artifacts" / "app.jar").exists() + + @mock.patch.object(kubernetes_commands, "run_command") + def test_container_mode_runs_in_docker(self, mock_run, tmp_path, java_example): + _lang_sdk_build_java_jar(tmp_path, None, native=False) + + assert all(call.args[0][0] == "docker" for call in mock_run.call_args_list) + + +class TestSetupLangSdkTestNativeSelection: + @pytest.mark.parametrize( + ("env_value", "expected_native"), + [("true", True), ("True", True), ("false", False), ("", False), (None, False)], + ) + def test_native_flag_is_read_from_env(self, monkeypatch, env_value, expected_native): + if env_value is None: + monkeypatch.delenv("LANG_SDK_NATIVE_TOOLCHAIN", raising=False) + else: + monkeypatch.setenv("LANG_SDK_NATIVE_TOOLCHAIN", env_value) + + captured: dict[str, bool] = {} + + def fake_parallel(steps, output): + for _title, thunk in steps: + thunk(None) + + monkeypatch.setattr(kubernetes_commands, "_run_lang_sdk_parallel", fake_parallel) + monkeypatch.setattr( + kubernetes_commands, + "_lang_sdk_build_go_bundle", + lambda staging, output, *, native: captured.update(go=native), + ) + monkeypatch.setattr( + kubernetes_commands, + "_lang_sdk_build_java_jar", + lambda staging, output, *, native: captured.update(java=native), + ) + for name in ( + "_lang_sdk_deploy_localstack", + "_lang_sdk_build_java_worker_image", + "_lang_sdk_upload_artifacts", + "_lang_sdk_apply_configmaps_and_secret", + "_lang_sdk_deploy_airflow", + ): + monkeypatch.setattr(kubernetes_commands, name, lambda *a, **k: None) + monkeypatch.setattr( + kubernetes_commands, + "BuildProdParams", + lambda python: mock.Mock(airflow_image_kubernetes="img"), + ) + + kubernetes_commands._setup_lang_sdk_test(python="3.10", kubernetes_version="v1.35.0") + + assert captured == {"go": expected_native, "java": expected_native} diff --git a/kubernetes-tests/lang_sdk/README.md b/kubernetes-tests/lang_sdk/README.md index 9444512b52fd1..eccf101649bb4 100644 --- a/kubernetes-tests/lang_sdk/README.md +++ b/kubernetes-tests/lang_sdk/README.md @@ -101,3 +101,10 @@ deploy, then runs the test. The `k8s-tests.yml` workflow enables it (`RUN_LANG_S which `--lang-sdk-test` reads) for a single variant only -- KubernetesExecutor with standard-naming off -- so the other five k8s jobs skip the test. The provisioning builds (Go bundle, Java jar, Java worker image) and the localstack deploy run in parallel. + +By default the Go bundle and Java jar are built inside ephemeral toolchain containers so a dev host +needs neither Go nor a JDK installed. In CI that variant sets `LANG_SDK_NATIVE_TOOLCHAIN=true`, which +makes breeze build both artifacts with the host `go` / `./gradlew` instead: the workflow provisions +the toolchains via `actions/setup-go` and `actions/setup-java` (whose built-in caches persist the Go +module/build cache and the Gradle distribution + dependency cache across runs), so the build skips the +per-run toolchain-image pulls and cold dependency downloads. From 4c8f2fc4df3a3f14b34ed8c6534cfe022128489d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 5 Jul 2026 12:48:01 +0200 Subject: [PATCH 2/2] Centralize lang-SDK JDK version and add a bustable cache key in CI Follow-ups to the native lang-SDK build: - Move the JDK version (17) into a JAVA_SDK_VERSION breeze constant and surface it as the java-sdk-version selective-checks / build-info output, so the k8s workflow reads it instead of hardcoding the version in YAML. - Restore the Go module/build and Gradle caches with an explicit actions/cache keyed on a "-v1-" salt rather than the setup-* built-in caching, giving an in-repo knob to force-invalidate a poisoned cache (bump the salt) without waiting for a dependency change to rotate it. --- .github/workflows/ci-amd.yml | 2 + .github/workflows/ci-arm.yml | 2 + .github/workflows/k8s-tests.yml | 39 +++++++++++++++---- dev/breeze/doc/ci/04_selective_checks.md | 1 + .../src/airflow_breeze/global_constants.py | 4 ++ .../airflow_breeze/utils/selective_checks.py | 2 + dev/breeze/tests/test_selective_checks.py | 13 +++++++ kubernetes-tests/lang_sdk/README.md | 11 ++++-- 8 files changed, 63 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 98a675ae2a844..750c430b780cf 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -99,6 +99,7 @@ jobs: include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} individual-providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} + java-sdk-version: ${{ steps.selective-checks.outputs.java-sdk-version }} kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} kubernetes-combos-list-as-string: >- ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} @@ -932,6 +933,7 @@ jobs: use-uv: ${{ needs.build-info.outputs.use-uv }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} + java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }} if: > ( needs.build-info.outputs.run-kubernetes-tests == 'true' || needs.build-info.outputs.run-helm-tests == 'true') diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index cebc66ff013c0..56385406a76a1 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -92,6 +92,7 @@ jobs: include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} individual-providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} + java-sdk-version: ${{ steps.selective-checks.outputs.java-sdk-version }} kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} kubernetes-combos-list-as-string: >- ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} @@ -925,6 +926,7 @@ jobs: use-uv: ${{ needs.build-info.outputs.use-uv }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} + java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }} if: > ( needs.build-info.outputs.run-kubernetes-tests == 'true' || needs.build-info.outputs.run-helm-tests == 'true') diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 8179f11d086d5..613357004abab 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -48,6 +48,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to debug resources" required: true type: string + java-sdk-version: + description: "JDK version used to build the lang-SDK Java artifacts natively in CI" + required: true + type: string permissions: contents: read jobs: @@ -96,24 +100,45 @@ jobs: make-mnt-writeable-and-cleanup: true id: breeze # preparing k8s environment with uv takes < 15 seconds with `uv` - there is no point in caching it. - # Provision the lang-SDK Go/Java toolchains on the host (with actions/setup-* built-in caching) - # only for the single variant that runs the lang-SDK test (KubernetesExecutor, standard-naming - # off). LANG_SDK_NATIVE_TOOLCHAIN=true then makes breeze build the artifacts natively instead of - # in throwaway toolchain containers, skipping the image pulls and reusing the cached toolchains. + # Provision the lang-SDK Go/Java toolchains on the host only for the single variant that runs the + # lang-SDK test (KubernetesExecutor, standard-naming off). LANG_SDK_NATIVE_TOOLCHAIN=true then makes + # breeze build the artifacts natively instead of in throwaway toolchain containers, skipping the + # image pulls and reusing the caches restored below. The module/build and Gradle caches are keyed + # explicitly (rather than via the setup-* built-in caching) so the key carries an ``-vN-`` salt: + # bump it to force-invalidate a poisoned cache without waiting for a dependency change to rotate it. # keep go-version in sync with go-sdk/go.mod and the sibling Go SDK job in ci-amd.yml. - name: "Setup Go for lang-SDK build" if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version: 1.25 - cache-dependency-path: kubernetes-tests/lang_sdk/go_example/go.sum + cache: false + - name: "Cache lang-SDK Go module + build cache" + if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: lang-sdk-go-v1-${{ runner.os }}-${{ hashFiles('kubernetes-tests/lang_sdk/go_example/go.sum') }} + restore-keys: | + lang-sdk-go-v1-${{ runner.os }}- - name: "Setup Java for lang-SDK build" if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 with: distribution: 'temurin' - java-version: '17' - cache: 'gradle' + java-version: ${{ inputs.java-sdk-version }} + - name: "Cache lang-SDK Gradle distribution + dependencies" + if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: lang-sdk-gradle-v1-${{ runner.os }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle-wrapper.properties', 'kubernetes-tests/lang_sdk/java_example/**/*.gradle*') }} # yamllint disable-line rule:line-length + restore-keys: | + lang-sdk-gradle-v1-${{ runner.os }}- - name: "\ Run complete K8S tests ${{ matrix.executor }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-\ ${{env.KUBERNETES_VERSION}}-${{ matrix.use-standard-naming }}" diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 67374350dedd0..eecf35b853cd6 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -531,6 +531,7 @@ GitHub Actions to pass the list of parameters to a command to execute | individual-providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (individually listed) | Providers[\amazon\] Providers\[google\] | * | | is-committer-build | Whether the build is triggered by a committer | false | | | is-legacy-ui-api-labeled | Whether the PR is labeled as legacy UI/API | false | | +| java-sdk-version | JDK version used to build the lang-SDK Java artifacts natively in CI | 17 | | | kind-version | Which Kind version to use for tests | v0.24.0 | | | kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.10-v1.25.2 3.11-v1.28.13 | * | | kubernetes-versions | All Kubernetes versions to use for tests as JSON array | \['v1.25.2'\] | | diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 38ccf343cedee..da34091a36d4c 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -229,6 +229,10 @@ JAVA_SDK = "java" ALLOWED_SDKS = [JAVA_SDK] +# JDK version used to build the Java SDK and its example bundles (e.g. the lang-SDK k8s system test). +# Keep in sync with the toolchain the Java SDK Gradle build targets. +JAVA_SDK_VERSION = "17" + DEFAULT_ALLOWED_EXECUTOR = ALLOWED_EXECUTORS[0] ALLOWED_AUTH_MANAGERS = [SIMPLE_AUTH_MANAGER, FAB_AUTH_MANAGER] START_AIRFLOW_ALLOWED_EXECUTORS = [LOCAL_EXECUTOR, CELERY_EXECUTOR, EDGE_EXECUTOR] diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 7c031c14014a7..0648b83515c79 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -44,6 +44,7 @@ DISABLE_TESTABLE_INTEGRATIONS_FROM_ARM, DISABLE_TESTABLE_INTEGRATIONS_FROM_CI, HELM_VERSION, + JAVA_SDK_VERSION, KIND_VERSION, NUMBER_OF_CORE_SLICES, NUMBER_OF_LOW_DEP_SLICES, @@ -695,6 +696,7 @@ def __str__(self) -> str: default_kubernetes_version = DEFAULT_KUBERNETES_VERSION default_kind_version = KIND_VERSION default_helm_version = HELM_VERSION + java_sdk_version = JAVA_SDK_VERSION @cached_property def latest_versions_only(self) -> bool: diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 552e0db77ed6e..cbecbdd94429d 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -29,6 +29,7 @@ ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, DEFAULT_KUBERNETES_VERSION, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + JAVA_SDK_VERSION, NUMBER_OF_CORE_SLICES, NUMBER_OF_LOW_DEP_SLICES, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, @@ -1643,6 +1644,18 @@ def test_ktlint_hook_only_runs_for_java_sdk_changes(files: tuple[str, ...], ktli assert ("ktlint" in skipped_hooks) is ktlint_skipped +def test_java_sdk_version_is_emitted_as_output(): + # The lang-SDK k8s job reads this to pick the JDK for the native Java build via actions/setup-java. + stderr = SelectiveChecks( + files=("README.md",), + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.PULL_REQUEST, + pr_labels=tuple(), + default_branch="main", + ) + assert get_outputs_from_stderr(str(stderr))["java-sdk-version"] == JAVA_SDK_VERSION + + @pytest.mark.skipif( not (AIRFLOW_ROOT_PATH / ".git").exists(), reason="This test should not run if .git folder is missing (for example by default in breeze container)", diff --git a/kubernetes-tests/lang_sdk/README.md b/kubernetes-tests/lang_sdk/README.md index eccf101649bb4..c43a14f07dc84 100644 --- a/kubernetes-tests/lang_sdk/README.md +++ b/kubernetes-tests/lang_sdk/README.md @@ -104,7 +104,10 @@ worker image) and the localstack deploy run in parallel. By default the Go bundle and Java jar are built inside ephemeral toolchain containers so a dev host needs neither Go nor a JDK installed. In CI that variant sets `LANG_SDK_NATIVE_TOOLCHAIN=true`, which -makes breeze build both artifacts with the host `go` / `./gradlew` instead: the workflow provisions -the toolchains via `actions/setup-go` and `actions/setup-java` (whose built-in caches persist the Go -module/build cache and the Gradle distribution + dependency cache across runs), so the build skips the -per-run toolchain-image pulls and cold dependency downloads. +makes breeze build both artifacts with the host `go` / `./gradlew` instead: the workflow installs the +toolchains via `actions/setup-go` and `actions/setup-java` and restores the Go module/build cache and +the Gradle distribution + dependency cache with `actions/cache`, so the build skips the per-run +toolchain-image pulls and cold dependency downloads. The cache keys carry a `-v1-` salt (see +`lang-sdk-go-v1-` / `lang-sdk-gradle-v1-` in `k8s-tests.yml`) — bump it to force-invalidate a poisoned +cache. The JDK version comes from the `java-sdk-version` build-info output (the `JAVA_SDK_VERSION` +breeze constant).