From edb597702b0eeebb4163c59660776989a58dcd27 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 18 Jun 2026 10:59:52 +0900 Subject: [PATCH 1/4] Add Go + Java + Python lang-SDK coordinator system test on KubernetesExecutor Adds a kind-based system test that runs one Dag (lang_sdk_combined) whose tasks span the Python task-runner and the Go/Java coordinators on the KubernetesExecutor. The golang/java queues route to their coordinators, each carrying a pod_template_file (the worktree-1 feature). Each worker pod's init container stages the queue's artifact (Go binary / Java jar) from a per-language localstack S3 bucket through the DagBundle interface into a shared emptyDir that is the coordinator's executables_root/jars_root. The Go queue runs on the prod image (self-contained static binary); the Java queue runs on the CI image (bundles the JVM the JavaCoordinator needs). New k8s-test-only bundles: go-sdk/example/k8s_combined and java-sdk/k8s-example. Provisioning is a new breeze command, `breeze k8s setup-lang-sdk-test`, which builds the artifacts, deploys localstack, uploads artifacts and the Python stub Dag, renders the pod-template image placeholders, and installs the Helm release. --- dev/breeze/doc/05_test_commands.rst | 17 + dev/breeze/doc/images/output_k8s.svg | 38 +- dev/breeze/doc/images/output_k8s.txt | 2 +- .../images/output_k8s_setup-lang-sdk-test.svg | 152 +++++++ .../images/output_k8s_setup-lang-sdk-test.txt | 1 + ...utput_setup_check-all-params-in-groups.svg | 4 +- ...utput_setup_check-all-params-in-groups.txt | 2 +- ...output_setup_regenerate-command-images.svg | 8 +- ...output_setup_regenerate-command-images.txt | 2 +- .../commands/kubernetes_commands.py | 381 ++++++++++++++++++ .../commands/kubernetes_commands_config.py | 13 +- go-sdk/example/k8s_combined/main.go | 89 ++++ java-sdk/k8s-example/build.gradle | 50 +++ java-sdk/k8s-example/gradle.properties | 19 + java-sdk/k8s-example/settings.gradle | 33 ++ .../airflow/k8sexample/CombinedExample.java | 47 +++ .../airflow/k8sexample/K8sBundleBuilder.java | 39 ++ kubernetes-tests/lang_sdk/Dockerfile.java | 31 ++ kubernetes-tests/lang_sdk/README.md | 95 +++++ kubernetes-tests/lang_sdk/config/values.yaml | 60 +++ .../lang_sdk/dags/lang_sdk_combined.py | 80 ++++ .../lang_sdk/manifests/localstack.yaml | 77 ++++ .../pod_templates/lang_sdk_golang.yaml | 117 ++++++ .../lang_sdk/pod_templates/lang_sdk_java.yaml | 115 ++++++ kubernetes-tests/lang_sdk/stage_artifacts.py | 92 +++++ .../test_lang_sdk_coordinator_executor.py | 81 ++++ 26 files changed, 1626 insertions(+), 19 deletions(-) create mode 100644 dev/breeze/doc/images/output_k8s_setup-lang-sdk-test.svg create mode 100644 dev/breeze/doc/images/output_k8s_setup-lang-sdk-test.txt create mode 100644 go-sdk/example/k8s_combined/main.go create mode 100644 java-sdk/k8s-example/build.gradle create mode 100644 java-sdk/k8s-example/gradle.properties create mode 100644 java-sdk/k8s-example/settings.gradle create mode 100644 java-sdk/k8s-example/src/java/org/apache/airflow/k8sexample/CombinedExample.java create mode 100644 java-sdk/k8s-example/src/java/org/apache/airflow/k8sexample/K8sBundleBuilder.java create mode 100644 kubernetes-tests/lang_sdk/Dockerfile.java create mode 100644 kubernetes-tests/lang_sdk/README.md create mode 100644 kubernetes-tests/lang_sdk/config/values.yaml create mode 100644 kubernetes-tests/lang_sdk/dags/lang_sdk_combined.py create mode 100644 kubernetes-tests/lang_sdk/manifests/localstack.yaml create mode 100644 kubernetes-tests/lang_sdk/pod_templates/lang_sdk_golang.yaml create mode 100644 kubernetes-tests/lang_sdk/pod_templates/lang_sdk_java.yaml create mode 100644 kubernetes-tests/lang_sdk/stage_artifacts.py create mode 100644 kubernetes-tests/tests/kubernetes_tests/test_lang_sdk_coordinator_executor.py diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index 9422918e9bfdb..ae165365164eb 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -565,6 +565,23 @@ All parameters of the command are here: :width: 100% :alt: Breeze k8s deploy-airflow +Setting up the lang-SDK coordinator system test +............................................... + +``breeze k8s setup-lang-sdk-test`` provisions a cluster for the lang-SDK coordinator +system test: it builds the Go and Java example bundles, deploys an in-cluster localstack +S3, uploads the artifacts and the Python stub Dag to their buckets, renders the +coordinator pod-template image placeholders, and installs the Helm release configured for +the ``golang`` and ``java`` queues. After it completes, run the test with +``breeze k8s tests``. + +All parameters of the command are here: + +.. image:: ./images/output_k8s_setup-lang-sdk-test.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_k8s_setup-lang-sdk-test.svg + :width: 100% + :alt: Breeze k8s setup-lang-sdk-test + Hot-reloading Dags and core sources ................................... diff --git a/dev/breeze/doc/images/output_k8s.svg b/dev/breeze/doc/images/output_k8s.svg index fd164ea3575ff..a76c947f06e72 100644 --- a/dev/breeze/doc/images/output_k8s.svg +++ b/dev/breeze/doc/images/output_k8s.svg @@ -1,4 +1,4 @@ - +