From 256a4c72a522b050c4a97ac7cab7fdadd1d3c25f Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 15 Jul 2026 08:08:29 +0000 Subject: [PATCH 1/2] Build Go SDK e2e example bundle natively in CI --- .github/workflows/airflow-e2e-tests.yml | 52 +++++++--- .../tests/airflow_e2e_tests/conftest.py | 97 +++++++++++-------- .../tests/airflow_e2e_tests/constants.py | 3 +- 3 files changed, 102 insertions(+), 50 deletions(-) diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 3799802a5db72..5f6ca25790656 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -121,16 +121,17 @@ jobs: use-uv: ${{ inputs.use-uv }} make-mnt-writeable-and-cleanup: true id: breeze - # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips the - # steps below. LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s - # job uses) makes the e2e conftest build with the host toolchain provisioned - # here instead of ephemeral eclipse-temurin containers, skipping the - # toolchain-image pull; local runs keep the containerized build. Without the - # cache each run re-downloads the example bundles' full dependency tree (Spark - # alone is ~1GB) into a fresh ~/.gradle. The key carries a ``-vN-`` salt (bump - # to force-invalidate a poisoned cache) and runner.arch keys the cache per - # architecture: ~/.gradle/jdks holds arch-specific auto-provisioned JDKs (the - # example pins a Java 11 toolchain) and this job runs on amd64 and arm64. + # The java_sdk mode runs the Gradle builds and the go_sdk mode runs the Go + # build; every other e2e mode skips the toolchain steps below. + # LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s job uses) + # makes the e2e conftest build with the host toolchain provisioned here + # instead of an ephemeral toolchain container, skipping the image pull; local + # runs keep the containerized builds. The cache keys carry a ``-vN-`` salt + # (bump to force-invalidate a poisoned cache) and runner.arch keys each cache + # per architecture, as this job runs on amd64 and arm64. Without its cache + # each java_sdk run re-downloads the example bundles' full dependency tree + # (Spark alone is ~1GB) into a fresh ~/.gradle, whose ~/.gradle/jdks holds + # arch-specific auto-provisioned JDKs (the example pins a Java 11 toolchain). - name: "Setup Java for the Java SDK build" if: inputs.e2e_test_mode == 'java_sdk' uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 @@ -151,12 +152,32 @@ jobs: key: e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line rule:line-length restore-keys: | e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}- + # go-version-file sources the Go toolchain from go-sdk/go.mod, so there is no + # separate version pin to keep in sync; cache: false because the explicit + # restore/save pair below persists the module + build caches even on red runs. + - name: "Setup Go for the Go SDK build" + if: inputs.e2e_test_mode == 'go_sdk' + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go-sdk/go.mod + cache: false + - name: "Restore Go SDK module + build cache" + if: inputs.e2e_test_mode == 'go_sdk' + id: go-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go-sdk/go.sum') }} + restore-keys: | + e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}- - name: "Test e2e integration tests" run: breeze testing airflow-e2e-tests env: DOCKER_IMAGE: "${{ inputs.docker-image-tag }}" E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}" - LANG_SDK_NATIVE_TOOLCHAIN: "${{ inputs.e2e_test_mode == 'java_sdk' && 'true' || 'false' }}" + LANG_SDK_NATIVE_TOOLCHAIN: "${{ (inputs.e2e_test_mode == 'java_sdk' || inputs.e2e_test_mode == 'go_sdk') && 'true' || 'false' }}" # yamllint disable-line rule:line-length - name: "Save Java SDK Gradle dependency cache" # Saved even when the e2e tests fail: the Gradle warm-up is independent of # test outcome, and actions/cache's post step would drop it on every red run. @@ -170,6 +191,15 @@ jobs: !~/.gradle/caches/*.lock !~/.gradle/caches/journal-1 key: ${{ steps.gradle-cache.outputs.cache-primary-key }} + - name: "Save Go SDK module + build cache" + # Saved even when the e2e tests fail, for the same reason as the Gradle cache above. + if: always() && inputs.e2e_test_mode == 'go_sdk' && steps.go-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ steps.go-cache.outputs.cache-primary-key }} - name: Zip logs run: | cd ./airflow-e2e-tests && zip -r logs.zip logs diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 56b51992bb4cb..3adf1e294201d 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -44,6 +44,7 @@ GO_SDK_BUNDLE_NAME, GO_SDK_DAGS_PATH, GO_SDK_EXAMPLE_BUNDLE_PKG, + GO_SDK_ROOT_PATH, JAVA_COMPOSE_PATH, JAVA_DOCKERFILE_PATH, JAVA_SDK_EXAMPLE_DAGS_PATH, @@ -499,53 +500,73 @@ def _setup_go_sdk_integration(dot_env_file, tmp_dir): ``CGO_ENABLED=0``), so the stock Airflow worker image can exec it directly without a Go toolchain or any extra runtime installed -- see ``go.yml``. """ - # Build + pack the example bundle inside an ephemeral Go container so the - # host does not need Go installed. + # `go tool airflow-go-pack` builds the bundle package, reads its + # --airflow-metadata, and appends the source + airflow-metadata.yaml + the + # AFBNDL01 trailer, writing a single self-contained executable bundle. + # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. + # + # In native mode (used in CI, where the host already has a Go toolchain plus + # restored module/build caches via actions/setup-go) the host `go` runs the + # build directly, skipping the toolchain-image pull and the container + # workarounds below. The containerized path stays the default for local runs + # so a dev host needs no Go installed: # # --user keeps build outputs owned by the current user (not root). # HOME points at a writable, gitignored dir under go-sdk/bin so the Go build # and module caches persist between runs (first run downloads modules once; # subsequent runs skip straight to compilation). - # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. # USER/HOME must be set because the SDK calls user.Current() at init; with # cgo disabled Go's pure-Go resolver reads those env vars instead of libc, # and panics if either is empty (the same vars are set on the worker in # go.yml so the packed binary runs the same way at execution time). - # `go tool airflow-go-pack` builds the bundle package, reads its - # --airflow-metadata, and appends the source + airflow-metadata.yaml + the - # AFBNDL01 trailer, writing a single self-contained executable bundle. - go_cache_home = "/repo/go-sdk/bin/.home" - bundle_out = f"/repo/go-sdk/bin/{GO_SDK_BUNDLE_NAME}" - console.print(f"[yellow]Building Go SDK example bundle ({GO_BUILDER_IMAGE})...") - subprocess.run( - [ - "docker", - "run", - "--rm", - "--user", - f"{os.getuid()}:{os.getgid()}", - "-e", - f"HOME={go_cache_home}", - "-e", - "USER=airflow", - "-e", - "CGO_ENABLED=0", - # Mount the repo so the whole go-sdk module (go.mod, tool directive, - # example sources) is visible to `go tool`. - "-v", - f"{AIRFLOW_ROOT_PATH}:/repo", - "-w", - "/repo/go-sdk", - GO_BUILDER_IMAGE, - "go", - "tool", - "airflow-go-pack", - "--output", - bundle_out, - GO_SDK_EXAMPLE_BUNDLE_PKG, - ], - check=True, - ) + if LANG_SDK_NATIVE_TOOLCHAIN: + console.print("[yellow]Building Go SDK example bundle (host toolchain)...") + subprocess.run( + [ + "go", + "tool", + "airflow-go-pack", + "--output", + str(GO_SDK_BIN_PATH / GO_SDK_BUNDLE_NAME), + GO_SDK_EXAMPLE_BUNDLE_PKG, + ], + cwd=GO_SDK_ROOT_PATH, + env={**os.environ, "CGO_ENABLED": "0"}, + check=True, + ) + else: + go_cache_home = "/repo/go-sdk/bin/.home" + bundle_out = f"/repo/go-sdk/bin/{GO_SDK_BUNDLE_NAME}" + console.print(f"[yellow]Building Go SDK example bundle ({GO_BUILDER_IMAGE})...") + subprocess.run( + [ + "docker", + "run", + "--rm", + "--user", + f"{os.getuid()}:{os.getgid()}", + "-e", + f"HOME={go_cache_home}", + "-e", + "USER=airflow", + "-e", + "CGO_ENABLED=0", + # Mount the repo so the whole go-sdk module (go.mod, tool directive, + # example sources) is visible to `go tool`. + "-v", + f"{AIRFLOW_ROOT_PATH}:/repo", + "-w", + "/repo/go-sdk", + GO_BUILDER_IMAGE, + "go", + "tool", + "airflow-go-pack", + "--output", + bundle_out, + GO_SDK_EXAMPLE_BUNDLE_PKG, + ], + check=True, + ) # Copy the compose override into the temp directory. copyfile(GO_COMPOSE_PATH, tmp_dir / "go.yml") diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py index ec1732ac75a8b..6b8d7c42d0c83 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py @@ -83,7 +83,8 @@ # Where airflow-go-pack writes the packed bundle inside the repo (go-sdk/bin is gitignored). GO_SDK_BIN_PATH = GO_SDK_ROOT_PATH / "bin" GO_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "go.yml" -# Go toolchain image used to build the bundle; must satisfy go-sdk/go.mod's toolchain. +# Go toolchain image used to build the bundle in the containerized path (i.e. unless +# LANG_SDK_NATIVE_TOOLCHAIN is set); must satisfy go-sdk/go.mod's toolchain. # The Alpine variant is ~7x smaller than the Debian one and is safe here because the # bundle is built with CGO_ENABLED=0 (a fully static binary, independent of musl/glibc) # and module fetches go through the HTTPS proxy (no git/gcc needed). From f70bd120b02569c90353c20b6c4efb03eb8df489 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Sat, 18 Jul 2026 13:42:51 +0000 Subject: [PATCH 2/2] Capture build output and remove path duplication in Go SDK e2e bundle build The native/containerized build branch was inlined with no output capture, so a failed CI build surfaced only a bare traceback, unlike the Java SDK's helper which prints the build log on failure. The containerized path also hardcoded /repo/go-sdk/bin instead of deriving it from the already-imported GO_SDK_ROOT_PATH/GO_SDK_BIN_PATH constants, and the Go build cache key ignored go.mod so a toolchain bump alone wouldn't invalidate it. --- .github/workflows/airflow-e2e-tests.yml | 2 +- .../tests/airflow_e2e_tests/conftest.py | 151 ++++++++++-------- 2 files changed, 85 insertions(+), 68 deletions(-) diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 5f6ca25790656..8cb55568e3e5b 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -169,7 +169,7 @@ jobs: path: | ~/.cache/go-build ~/go/pkg/mod - key: e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go-sdk/go.sum') }} + key: e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go-sdk/go.mod', 'go-sdk/go.sum') }} # yamllint disable-line rule:line-length restore-keys: | e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}- - name: "Test e2e integration tests" diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 3adf1e294201d..72687795ae1f9 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -488,6 +488,89 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): os.environ["ENV_FILE_PATH"] = str(dot_env_file) +def _run_go_sdk_pack(output_path, *, capture_output=False, native=False): + """Run ``go tool airflow-go-pack`` natively or inside the pinned Go toolchain container. + + ``go tool airflow-go-pack`` builds the bundle package, reads its + --airflow-metadata, and appends the source + airflow-metadata.yaml + the + AFBNDL01 trailer, writing a single self-contained executable bundle. + CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. + + In ``native`` mode (used in CI, where the host already has a Go toolchain plus + restored module/build caches via ``actions/setup-go``) it invokes the host ``go`` + directly, skipping the toolchain-image pull and the container workarounds below. + + The containerized path stays the default for local runs so a dev host needs + no Go installed: + + * --user keeps build outputs owned by the current user (not root). + * HOME points at a writable, gitignored dir under go-sdk/bin so the Go build + and module caches persist between runs (first run downloads modules once; + subsequent runs skip straight to compilation). + * USER/HOME must be set because the SDK calls user.Current() at init; with + cgo disabled Go's pure-Go resolver reads those env vars instead of libc, + and panics if either is empty (the same vars are set on the worker in + go.yml so the packed binary runs the same way at execution time). + """ + if native: + cwd = GO_SDK_ROOT_PATH + env = {**os.environ, "CGO_ENABLED": "0"} + argv = [ + "go", + "tool", + "airflow-go-pack", + "--output", + str(output_path), + GO_SDK_EXAMPLE_BUNDLE_PKG, + ] + else: + cwd = None + env = None + # Mount the repo so the whole go-sdk module (go.mod, tool directive, + # example sources) is visible to `go tool`. + container_go_sdk_dir = f"/repo/{GO_SDK_ROOT_PATH.relative_to(AIRFLOW_ROOT_PATH)}" + container_bin_dir = f"/repo/{GO_SDK_BIN_PATH.relative_to(AIRFLOW_ROOT_PATH)}" + argv = [ + "docker", + "run", + "--rm", + "--user", + f"{os.getuid()}:{os.getgid()}", + "-e", + f"HOME={container_bin_dir}/.home", + "-e", + "USER=airflow", + "-e", + "CGO_ENABLED=0", + "-v", + f"{AIRFLOW_ROOT_PATH}:/repo", + "-w", + container_go_sdk_dir, + GO_BUILDER_IMAGE, + "go", + "tool", + "airflow-go-pack", + "--output", + f"{container_bin_dir}/{output_path.name}", + GO_SDK_EXAMPLE_BUNDLE_PKG, + ] + return subprocess.run(argv, cwd=cwd, env=env, check=True, capture_output=capture_output, text=True) + + +def _pack_go_sdk_example_bundle(*, native=False): + """Build the Go SDK example bundle, capturing output so a failure prints the build log.""" + output_path = GO_SDK_BIN_PATH / GO_SDK_BUNDLE_NAME + mode_label = "host toolchain" if native else GO_BUILDER_IMAGE + console.print(f"[yellow]Building Go SDK example bundle ({mode_label})...") + try: + completed = _run_go_sdk_pack(output_path, capture_output=True, native=native) + except subprocess.CalledProcessError as e: + console.print("[red]Go SDK example bundle build failed:") + console.print(e.stdout, e.stderr, sep="\n", markup=False, soft_wrap=True) + raise + console.print(completed.stdout, completed.stderr, sep="\n", markup=False, soft_wrap=True) + + def _setup_go_sdk_integration(dot_env_file, tmp_dir): """Set up the go_sdk E2E test mode. @@ -500,73 +583,7 @@ def _setup_go_sdk_integration(dot_env_file, tmp_dir): ``CGO_ENABLED=0``), so the stock Airflow worker image can exec it directly without a Go toolchain or any extra runtime installed -- see ``go.yml``. """ - # `go tool airflow-go-pack` builds the bundle package, reads its - # --airflow-metadata, and appends the source + airflow-metadata.yaml + the - # AFBNDL01 trailer, writing a single self-contained executable bundle. - # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. - # - # In native mode (used in CI, where the host already has a Go toolchain plus - # restored module/build caches via actions/setup-go) the host `go` runs the - # build directly, skipping the toolchain-image pull and the container - # workarounds below. The containerized path stays the default for local runs - # so a dev host needs no Go installed: - # - # --user keeps build outputs owned by the current user (not root). - # HOME points at a writable, gitignored dir under go-sdk/bin so the Go build - # and module caches persist between runs (first run downloads modules once; - # subsequent runs skip straight to compilation). - # USER/HOME must be set because the SDK calls user.Current() at init; with - # cgo disabled Go's pure-Go resolver reads those env vars instead of libc, - # and panics if either is empty (the same vars are set on the worker in - # go.yml so the packed binary runs the same way at execution time). - if LANG_SDK_NATIVE_TOOLCHAIN: - console.print("[yellow]Building Go SDK example bundle (host toolchain)...") - subprocess.run( - [ - "go", - "tool", - "airflow-go-pack", - "--output", - str(GO_SDK_BIN_PATH / GO_SDK_BUNDLE_NAME), - GO_SDK_EXAMPLE_BUNDLE_PKG, - ], - cwd=GO_SDK_ROOT_PATH, - env={**os.environ, "CGO_ENABLED": "0"}, - check=True, - ) - else: - go_cache_home = "/repo/go-sdk/bin/.home" - bundle_out = f"/repo/go-sdk/bin/{GO_SDK_BUNDLE_NAME}" - console.print(f"[yellow]Building Go SDK example bundle ({GO_BUILDER_IMAGE})...") - subprocess.run( - [ - "docker", - "run", - "--rm", - "--user", - f"{os.getuid()}:{os.getgid()}", - "-e", - f"HOME={go_cache_home}", - "-e", - "USER=airflow", - "-e", - "CGO_ENABLED=0", - # Mount the repo so the whole go-sdk module (go.mod, tool directive, - # example sources) is visible to `go tool`. - "-v", - f"{AIRFLOW_ROOT_PATH}:/repo", - "-w", - "/repo/go-sdk", - GO_BUILDER_IMAGE, - "go", - "tool", - "airflow-go-pack", - "--output", - bundle_out, - GO_SDK_EXAMPLE_BUNDLE_PKG, - ], - check=True, - ) + _pack_go_sdk_example_bundle(native=LANG_SDK_NATIVE_TOOLCHAIN) # Copy the compose override into the temp directory. copyfile(GO_COMPOSE_PATH, tmp_dir / "go.yml")