diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 3799802a5db72..8cb55568e3e5b 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -121,16 +121,17 @@ jobs: use-uv: ${{ inputs.use-uv }} make-mnt-writeable-and-cleanup: true id: breeze - # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips the - # steps below. LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s - # job uses) makes the e2e conftest build with the host toolchain provisioned - # here instead of ephemeral eclipse-temurin containers, skipping the - # toolchain-image pull; local runs keep the containerized build. Without the - # cache each run re-downloads the example bundles' full dependency tree (Spark - # alone is ~1GB) into a fresh ~/.gradle. The key carries a ``-vN-`` salt (bump - # to force-invalidate a poisoned cache) and runner.arch keys the cache per - # architecture: ~/.gradle/jdks holds arch-specific auto-provisioned JDKs (the - # example pins a Java 11 toolchain) and this job runs on amd64 and arm64. + # The java_sdk mode runs the Gradle builds and the go_sdk mode runs the Go + # build; every other e2e mode skips the toolchain steps below. + # LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s job uses) + # makes the e2e conftest build with the host toolchain provisioned here + # instead of an ephemeral toolchain container, skipping the image pull; local + # runs keep the containerized builds. The cache keys carry a ``-vN-`` salt + # (bump to force-invalidate a poisoned cache) and runner.arch keys each cache + # per architecture, as this job runs on amd64 and arm64. Without its cache + # each java_sdk run re-downloads the example bundles' full dependency tree + # (Spark alone is ~1GB) into a fresh ~/.gradle, whose ~/.gradle/jdks holds + # arch-specific auto-provisioned JDKs (the example pins a Java 11 toolchain). - name: "Setup Java for the Java SDK build" if: inputs.e2e_test_mode == 'java_sdk' uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 @@ -151,12 +152,32 @@ jobs: key: e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line rule:line-length restore-keys: | e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}- + # go-version-file sources the Go toolchain from go-sdk/go.mod, so there is no + # separate version pin to keep in sync; cache: false because the explicit + # restore/save pair below persists the module + build caches even on red runs. + - name: "Setup Go for the Go SDK build" + if: inputs.e2e_test_mode == 'go_sdk' + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go-sdk/go.mod + cache: false + - name: "Restore Go SDK module + build cache" + if: inputs.e2e_test_mode == 'go_sdk' + id: go-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go-sdk/go.mod', 'go-sdk/go.sum') }} # yamllint disable-line rule:line-length + restore-keys: | + e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}- - name: "Test e2e integration tests" run: breeze testing airflow-e2e-tests env: DOCKER_IMAGE: "${{ inputs.docker-image-tag }}" E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}" - LANG_SDK_NATIVE_TOOLCHAIN: "${{ inputs.e2e_test_mode == 'java_sdk' && 'true' || 'false' }}" + LANG_SDK_NATIVE_TOOLCHAIN: "${{ (inputs.e2e_test_mode == 'java_sdk' || inputs.e2e_test_mode == 'go_sdk') && 'true' || 'false' }}" # yamllint disable-line rule:line-length - name: "Save Java SDK Gradle dependency cache" # Saved even when the e2e tests fail: the Gradle warm-up is independent of # test outcome, and actions/cache's post step would drop it on every red run. @@ -170,6 +191,15 @@ jobs: !~/.gradle/caches/*.lock !~/.gradle/caches/journal-1 key: ${{ steps.gradle-cache.outputs.cache-primary-key }} + - name: "Save Go SDK module + build cache" + # Saved even when the e2e tests fail, for the same reason as the Gradle cache above. + if: always() && inputs.e2e_test_mode == 'go_sdk' && steps.go-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ steps.go-cache.outputs.cache-primary-key }} - name: Zip logs run: | cd ./airflow-e2e-tests && zip -r logs.zip logs diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 56b51992bb4cb..72687795ae1f9 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -44,6 +44,7 @@ GO_SDK_BUNDLE_NAME, GO_SDK_DAGS_PATH, GO_SDK_EXAMPLE_BUNDLE_PKG, + GO_SDK_ROOT_PATH, JAVA_COMPOSE_PATH, JAVA_DOCKERFILE_PATH, JAVA_SDK_EXAMPLE_DAGS_PATH, @@ -487,65 +488,102 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): os.environ["ENV_FILE_PATH"] = str(dot_env_file) -def _setup_go_sdk_integration(dot_env_file, tmp_dir): - """Set up the go_sdk E2E test mode. +def _run_go_sdk_pack(output_path, *, capture_output=False, native=False): + """Run ``go tool airflow-go-pack`` natively or inside the pinned Go toolchain container. - Compiles the Go SDK example bundle into a self-contained executable bundle - via the ``airflow-go-pack`` tooling, drops it into the directory the - ``ExecutableCoordinator`` scans, copies the Python stub Dag, and writes the - coordinator configuration. + ``go tool airflow-go-pack`` builds the bundle package, reads its + --airflow-metadata, and appends the source + airflow-metadata.yaml + the + AFBNDL01 trailer, writing a single self-contained executable bundle. + CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. - The packed bundle is a statically linked native executable (built with - ``CGO_ENABLED=0``), so the stock Airflow worker image can exec it directly - without a Go toolchain or any extra runtime installed -- see ``go.yml``. + In ``native`` mode (used in CI, where the host already has a Go toolchain plus + restored module/build caches via ``actions/setup-go``) it invokes the host ``go`` + directly, skipping the toolchain-image pull and the container workarounds below. + + The containerized path stays the default for local runs so a dev host needs + no Go installed: + + * --user keeps build outputs owned by the current user (not root). + * HOME points at a writable, gitignored dir under go-sdk/bin so the Go build + and module caches persist between runs (first run downloads modules once; + subsequent runs skip straight to compilation). + * USER/HOME must be set because the SDK calls user.Current() at init; with + cgo disabled Go's pure-Go resolver reads those env vars instead of libc, + and panics if either is empty (the same vars are set on the worker in + go.yml so the packed binary runs the same way at execution time). """ - # Build + pack the example bundle inside an ephemeral Go container so the - # host does not need Go installed. - # - # --user keeps build outputs owned by the current user (not root). - # HOME points at a writable, gitignored dir under go-sdk/bin so the Go build - # and module caches persist between runs (first run downloads modules once; - # subsequent runs skip straight to compilation). - # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. - # USER/HOME must be set because the SDK calls user.Current() at init; with - # cgo disabled Go's pure-Go resolver reads those env vars instead of libc, - # and panics if either is empty (the same vars are set on the worker in - # go.yml so the packed binary runs the same way at execution time). - # `go tool airflow-go-pack` builds the bundle package, reads its - # --airflow-metadata, and appends the source + airflow-metadata.yaml + the - # AFBNDL01 trailer, writing a single self-contained executable bundle. - go_cache_home = "/repo/go-sdk/bin/.home" - bundle_out = f"/repo/go-sdk/bin/{GO_SDK_BUNDLE_NAME}" - console.print(f"[yellow]Building Go SDK example bundle ({GO_BUILDER_IMAGE})...") - subprocess.run( - [ + if native: + cwd = GO_SDK_ROOT_PATH + env = {**os.environ, "CGO_ENABLED": "0"} + argv = [ + "go", + "tool", + "airflow-go-pack", + "--output", + str(output_path), + GO_SDK_EXAMPLE_BUNDLE_PKG, + ] + else: + cwd = None + env = None + # Mount the repo so the whole go-sdk module (go.mod, tool directive, + # example sources) is visible to `go tool`. + container_go_sdk_dir = f"/repo/{GO_SDK_ROOT_PATH.relative_to(AIRFLOW_ROOT_PATH)}" + container_bin_dir = f"/repo/{GO_SDK_BIN_PATH.relative_to(AIRFLOW_ROOT_PATH)}" + argv = [ "docker", "run", "--rm", "--user", f"{os.getuid()}:{os.getgid()}", "-e", - f"HOME={go_cache_home}", + f"HOME={container_bin_dir}/.home", "-e", "USER=airflow", "-e", "CGO_ENABLED=0", - # Mount the repo so the whole go-sdk module (go.mod, tool directive, - # example sources) is visible to `go tool`. "-v", f"{AIRFLOW_ROOT_PATH}:/repo", "-w", - "/repo/go-sdk", + container_go_sdk_dir, GO_BUILDER_IMAGE, "go", "tool", "airflow-go-pack", "--output", - bundle_out, + f"{container_bin_dir}/{output_path.name}", GO_SDK_EXAMPLE_BUNDLE_PKG, - ], - check=True, - ) + ] + return subprocess.run(argv, cwd=cwd, env=env, check=True, capture_output=capture_output, text=True) + + +def _pack_go_sdk_example_bundle(*, native=False): + """Build the Go SDK example bundle, capturing output so a failure prints the build log.""" + output_path = GO_SDK_BIN_PATH / GO_SDK_BUNDLE_NAME + mode_label = "host toolchain" if native else GO_BUILDER_IMAGE + console.print(f"[yellow]Building Go SDK example bundle ({mode_label})...") + try: + completed = _run_go_sdk_pack(output_path, capture_output=True, native=native) + except subprocess.CalledProcessError as e: + console.print("[red]Go SDK example bundle build failed:") + console.print(e.stdout, e.stderr, sep="\n", markup=False, soft_wrap=True) + raise + console.print(completed.stdout, completed.stderr, sep="\n", markup=False, soft_wrap=True) + + +def _setup_go_sdk_integration(dot_env_file, tmp_dir): + """Set up the go_sdk E2E test mode. + + Compiles the Go SDK example bundle into a self-contained executable bundle + via the ``airflow-go-pack`` tooling, drops it into the directory the + ``ExecutableCoordinator`` scans, copies the Python stub Dag, and writes the + coordinator configuration. + + The packed bundle is a statically linked native executable (built with + ``CGO_ENABLED=0``), so the stock Airflow worker image can exec it directly + without a Go toolchain or any extra runtime installed -- see ``go.yml``. + """ + _pack_go_sdk_example_bundle(native=LANG_SDK_NATIVE_TOOLCHAIN) # Copy the compose override into the temp directory. copyfile(GO_COMPOSE_PATH, tmp_dir / "go.yml") diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py index ec1732ac75a8b..6b8d7c42d0c83 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py @@ -83,7 +83,8 @@ # Where airflow-go-pack writes the packed bundle inside the repo (go-sdk/bin is gitignored). GO_SDK_BIN_PATH = GO_SDK_ROOT_PATH / "bin" GO_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "go.yml" -# Go toolchain image used to build the bundle; must satisfy go-sdk/go.mod's toolchain. +# Go toolchain image used to build the bundle in the containerized path (i.e. unless +# LANG_SDK_NATIVE_TOOLCHAIN is set); must satisfy go-sdk/go.mod's toolchain. # The Alpine variant is ~7x smaller than the Debian one and is safe here because the # bundle is built with CGO_ENABLED=0 (a fully static binary, independent of musl/glibc) # and module fetches go through the HTTPS proxy (no git/gcc needed).