Go SDK: airflow-go-pack tool for self-contained bundle binaries

go-sdk/adr/0002-use-go-tool-directive-for-bundle-packer.md

@@ -113,35 +113,48 @@ convention so authors can forward arbitrary flags to the underlying
       to produce the *target artifact*.
    3. Executes a *host-runnable introspection binary* with
       `--airflow-metadata` to obtain `sdk.{language,version,supervisor_schema_version}`
-      and the `dags` mapping. The packer first tries to exec the target
-      artifact directly; if that fails with "exec format error" (the
-      target is built for a different OS/arch than the host), the
-      packer builds a host-arch sidecar from the same package
-      (`go build` with `GOOS`/`GOARCH` unset, written to
-      `<tmpdir>/<binname>.introspect`) and execs that instead. Both
-      binaries are produced from the same source package against the
-      same module graph, so `RegisterDags` records identical dag/task
-      identity regardless of which one is execed.
-   4. Writes a deterministic ZIP next to the working directory at
-      `<bundleName>.zip`, where `<bundleName>` comes from the
-      binary's `BundleInfo.Name` (part of the same `--airflow-metadata`
-      introspection output).
+      and the `dags` mapping. When cross-compiling, the packer builds a
+      host-arch sidecar from the same package and forwarded build flags
+      and execs that instead; both come from the same sources and flags,
+      so `RegisterDags` records identical identity either way. The sidecar
+      is skipped when `--airflow-metadata` supplies the manifest directly
+      (see the overrides below).
+   4. Writes a self-contained bundle: the deployable executable with
+      the source bytes, the manifest bytes, and the fixed 64-byte
+      `AFBNDL01` trailer appended (the format from ADR 0004 /
+      `task-sdk/docs/executable-bundle-spec.rst`). By default it is
+      written next to the working directory under the bundle's main
+      package directory name; `--output` overrides the path.
 
    Optional overrides, for advanced or pre-built workflows:
 
-   - `--source <path>`: override the auto-detected source file.
-   - `--executable <path>`: skip the internal `go build` and pack a
-     pre-built binary. Mutually exclusive with `--` build-flag
-     passthrough. If the supplied binary is not host-runnable (e.g.
-     the user cross-built a `linux/amd64` binary from a `darwin/arm64`
-     host), the packer still needs to introspect it: it builds a
-     host-arch sidecar from the positional package and execs that for
-     `--airflow-metadata`, then appends the resulting footer to the
-     user-supplied binary. If no positional package was passed and
-     the supplied binary is not host-runnable, the packer errors with
-     a message asking for the source package so the sidecar can be
-     built.
-   - `--output <path>`: override the default output ZIP path.
+   - `--source <path>`: override the auto-detected source file, and the
+     escape hatch when discovery can't pick it. It skips discovery
+     entirely (a plain `go list` that ignores build tags and
+     `GOOS`/`GOARCH`, so it can fail or pick the wrong file).
+   - `--executable <path>`: skip `go build` and pack a pre-built binary.
+     Mutually exclusive with `--` build flags and with `--goos`/`--goarch`
+     (it never builds). The binary must run on the host so the packer can
+     exec it for `--airflow-metadata`. A non-host-runnable binary is a hard
+     error: the packer won't rebuild a host sidecar, because the original
+     build inputs (tags, `-ldflags`, `GOOS`/`GOARCH` files) are unknown and
+     a rebuild could advertise a different dag/task set than shipped. To
+     pack for another platform, use the build path (`--goos`/`--goarch`,
+     below) or supply `--airflow-metadata`.
+   - `--airflow-metadata <path>`: supply a captured manifest instead of
+     introspecting the binary. Accepts the binary's YAML default, its
+     `--format json` output, or a bundle's embedded `airflow-metadata.yaml`
+     (one YAML decoder reads all three). Short-circuits introspection in
+     every mode — the deterministic way to pack a pre-built cross binary
+     with `--executable`.
+   - `--goos <os>` / `--goarch <arch>`: cross-compile the deployable bundle.
+     Prefer these over the `GOOS`/`GOARCH` env vars: under `go tool` those
+     env vars cross-build the packer itself, which then can't exec on the
+     host. The flags target only the internal `go build`, leaving the
+     packer build host-native; they fall back to the env vars, then the
+     host. Mutually exclusive with `--executable`.
+   - `--output <path>`: override the default output path. Its parent
+     directory is created if missing.
 
    Examples:
 
@@ -152,14 +165,20 @@ convention so authors can forward arbitrary flags to the underlying
    # Pack a different package, with extra go build flags.
    go tool airflow-go-pack ./cmd/my-bundle -- -trimpath -tags=prod
 
-   # Pack an already-built binary (skips go build).
+   # Pack an already-built binary for THIS host (skips go build).
    go tool airflow-go-pack --executable ./build/example --source main.go
+
+   # Pack a bundle for a different platform: cross-build via the build
+   # path (no --executable), forwarding go build flags after "--".
+   # Use --goos/--goarch, NOT the GOOS/GOARCH env vars, under `go tool`.
+   go tool airflow-go-pack --goos linux --goarch amd64 ./cmd/my-bundle -- -trimpath
    ```
 
 2. **Extend the existing `--airflow-metadata` flag in
    `bundlev1server.Serve` to print the full spec.** Rather than adding a
    second introspection flag, `--airflow-metadata` is the single flag the
-   packer relies on; it prints a JSON document of the form:
+   packer relies on; it prints a manifest document (YAML by default, JSON
+   under `--format json`) of the form (shown here as JSON for structure):
 
    ```json
    {
@@ -183,9 +202,18 @@ convention so authors can forward arbitrary flags to the underlying
    recording registry, then enumerating the recorded dags and their
    tasks. `--airflow-metadata` today prints only `BundleInfo`
    (`server.go`); it is extended to emit this full document, so the
-   shipped `decideMode` switch needs only one metadata mode. The bundle's
-   `BundleInfo.Name` (used by the packer for the default output
-   filename) is carried in the same output.
+   shipped `decideMode` switch needs only one metadata mode. The packer
+   derives the default output filename from the bundle's main package
+   directory name (what `go build` itself names the binary), resolved
+   before the build so a bad `--output` fails fast; it does not come from
+   `BundleInfo.Name`, and no name field is carried in this output.
+
+   A `--format yaml|json` flag selects the encoding and is only valid with
+   `--airflow-metadata` (misuse is a hard error). The default is YAML,
+   matching a bundle's embedded `airflow-metadata.yaml`, so
+   `mybundle --airflow-metadata > airflow-metadata.yaml` is ready to use.
+   The packer never sets `--format`; its YAML decoder reads both the YAML
+   default and `--format json` output (JSON is a subset of YAML).
 
 3. **Bundle authors register the packer in their own `go.mod`:**
 
@@ -251,12 +279,14 @@ convention so authors can forward arbitrary flags to the underlying
 - `go build` flag passthrough uses the standard `--` separator
   convention so the packer's own flag set stays small and stable.
 - Host-runnable detection is by attempted exec, not by parsing the
-  binary's exec format. The packer runs the candidate introspection
-  binary with `--airflow-metadata` and treats the OS's "exec format
-  error" (and the Windows equivalent surfaced by `os/exec`) as the
-  signal to fall back to building a host-arch sidecar. Other exec
-  failures (non-zero exit, malformed JSON, missing flag) are real
-  errors and are surfaced to the user as-is. The Go build cache
-  amortises the sidecar to a link step when host arch is already
-  involved, so there is no measurable overhead when no cross-compile
-  is in play.
+  binary's format: the packer runs the candidate with `--airflow-metadata`
+  and treats an "exec format error" (and its Windows equivalent) as
+  not-startable. Other failures (non-zero exit, malformed output, missing
+  flag) are surfaced as-is.
+- A not-startable binary is handled differently per mode. The **build**
+  path owns the build, so it builds a host-arch sidecar from the same
+  package and `--` flags (the build cache amortises this to a link step,
+  so no measurable overhead without a cross-compile). In **`--executable`**
+  mode the build inputs are unknown, so the packer must not synthesise a
+  sidecar that could diverge from the shipped binary: it fails fast and
+  points the user at the cross-build path or `--airflow-metadata`.

go-sdk/bundle/bundlev1/bundlev1server/server.go

@@ -18,9 +18,7 @@
 package bundlev1server
 
 import (
-	"encoding/json"
 	"errors"
-	"fmt"
 	"log/slog"
 	"os"
 
@@ -44,13 +42,28 @@ var ErrCoordinatorFlagsIncomplete = errors.New(
 	"--comm and --logs must be supplied together",
 )
 
-// CLI Flags.
-// The --bundle-metadata flag is used for showing the embedded bundle info in airflow-metadata.yaml spec format.
-// The --comm and --logs select the coordinator-mode protocol
-// All three are read by Serve to choose a server mode below.
+// ErrFormatRequiresMetadata is returned by [Serve] when --format is supplied
+// without --airflow-metadata, the only mode whose encoding it selects.
+var ErrFormatRequiresMetadata = errors.New(
+	"--format is only valid together with --airflow-metadata",
+)
+
+// CLI Flags, all read by Serve to choose a server mode below.
+// --airflow-metadata prints the bundle's manifest and exits (airflow-go-pack
+// consumes it to build the embedded airflow-metadata.yaml); --format selects
+// its encoding. --comm and --logs select coordinator mode.
 var (
-	versionInfo = flag.Bool("bundle-metadata", false, "show the embedded bundle info")
-	commAddr    = flag.String(
+	printMetadata = flag.Bool(
+		"airflow-metadata",
+		false,
+		"print the bundle's airflow-metadata manifest and exit",
+	)
+	metadataFormat = flag.String(
+		"format",
+		string(execution.MetadataFormatYAML),
+		"encoding for --airflow-metadata: yaml (default) or json; only valid with --airflow-metadata",
+	)
+	commAddr = flag.String(
 		"comm",
 		"",
 		"host:port of the supervisor's coordinator comm channel (selects coordinator mode)",
@@ -83,7 +96,7 @@ type serveMode int
 
 const (
 	modePlugin                serveMode = iota // go-plugin gRPC (existing Edge Worker path)
-	modeMetadataDump                           // --bundle-metadata: print BundleInfo JSON
+	modeAirflowMetadata                        // --airflow-metadata: print the manifest JSON (ADR 0002/0004)
 	modeCoordinator                            // --comm/--logs: msgpack-over-IPC (ADR 0003)
 	modeCoordinatorUsageError                  // misuse: print usage and exit non-zero
 )
@@ -114,9 +127,21 @@ func Serve(bundle bundlev1.BundleProvider, opts ...ServeOpt) error {
 		c.ApplyServeOpt(serveConfig)
 	}
 
-	switch decideMode() {
-	case modeMetadataDump:
-		return dumpBundleMetadata(bundle)
+	mode := decideMode()
+
+	// --format applies only to --airflow-metadata; reject it elsewhere instead
+	// of silently ignoring it.
+	if mode != modeAirflowMetadata && flag.CommandLine.Changed("format") {
+		return ErrFormatRequiresMetadata
+	}
+
+	switch mode {
+	case modeAirflowMetadata:
+		format, err := execution.ParseMetadataFormat(*metadataFormat)
+		if err != nil {
+			return err
+		}
+		return execution.DumpAirflowMetadata(bundle, format)
 	case modeCoordinator:
 		// In coordinator mode the supervisor reads the logs channel for
 		// structured records, so configuring the hclog/stderr default
@@ -133,8 +158,8 @@ func Serve(bundle bundlev1.BundleProvider, opts ...ServeOpt) error {
 }
 
 func decideMode() serveMode {
-	if *versionInfo {
-		return modeMetadataDump
+	if *printMetadata {
+		return modeAirflowMetadata
 	}
 	commSet := *commAddr != ""
 	logsSet := *logsAddr != ""
@@ -148,16 +173,6 @@ func decideMode() serveMode {
 	return modePlugin
 }
 
-func dumpBundleMetadata(bundle bundlev1.BundleProvider) error {
-	meta := bundle.GetBundleVersion()
-	data, err := json.MarshalIndent(meta, "", "    ")
-	if err != nil {
-		return err
-	}
-	fmt.Println(string(data))
-	return nil
-}
-
 func installPluginLogger() {
 	hcLogger := hclog.New(&hclog.LoggerOptions{
 		Level:                    hclog.Trace,

go-sdk/bundle/bundlev1/registry.go

@@ -43,9 +43,30 @@ type (
 		AddDag(dagId string) Dag
 	}
 
+	// TaskInfo describes a registered task by its user-visible id.
+	TaskInfo struct {
+		ID string
+	}
+
+	// DagInfo describes a registered dag together with its tasks in
+	// registration order.
+	DagInfo struct {
+		DagID string
+		Tasks []TaskInfo
+	}
+
+	// EnumerableBundle exposes the dag/task identity recorded by RegisterDags.
+	// The default registry implements it; airflow-go-pack relies on it to read
+	// a bundle's dag/task ids without executing any task.
+	EnumerableBundle interface {
+		OrderedDags() []DagInfo
+	}
+
 	registry struct {
 		sync.RWMutex
 		taskFuncMap map[string]map[string]Task
+		dagOrder    []string
+		taskOrder   map[string][]string
 	}
 )
 
@@ -64,7 +85,10 @@ func (d dagShim) AddTaskWithName(taskId string, fn any) {
 
 // Function New creates a new bundle on which Dag and Tasks can be registered
 func New() Registry {
-	return &registry{taskFuncMap: make(map[string]map[string]Task)}
+	return &registry{
+		taskFuncMap: make(map[string]map[string]Task),
+		taskOrder:   make(map[string][]string),
+	}
 }
 
 func getFnName(fn reflect.Value) string {
@@ -76,10 +100,14 @@ func getFnName(fn reflect.Value) string {
 }
 
 func (r *registry) AddDag(dagId string) Dag {
+	r.Lock()
+	defer r.Unlock()
+
 	if _, exists := r.taskFuncMap[dagId]; exists {
 		panic(fmt.Errorf("Dag %q already exists in bundle", dagId))
 	}
 	r.taskFuncMap[dagId] = make(map[string]Task)
+	r.dagOrder = append(r.dagOrder, dagId)
 	return dagShim{dagId, r}
 }
 
@@ -109,13 +137,15 @@ func (r *registry) registerTaskWithName(dagId, taskId string, fn any) {
 	if !exists {
 		dagTasks = make(map[string]Task)
 		r.taskFuncMap[dagId] = dagTasks
+		r.dagOrder = append(r.dagOrder, dagId)
 	}
 
 	_, exists = dagTasks[taskId]
 	if exists {
 		panic(fmt.Errorf("taskId %q is already registered for DAG %q", taskId, dagId))
 	}
 	dagTasks[taskId] = task
+	r.taskOrder[dagId] = append(r.taskOrder[dagId], taskId)
 }
 
 func (r *registry) LookupTask(dagId, taskId string) (task Task, exists bool) {
@@ -129,3 +159,22 @@ func (r *registry) LookupTask(dagId, taskId string) (task Task, exists bool) {
 	task, exists = dagTasks[taskId]
 	return task, exists
 }
+
+// OrderedDags returns the registered dags in AddDag order, each with its tasks
+// in registration order. The returned slice is freshly allocated; callers may
+// mutate it freely.
+func (r *registry) OrderedDags() []DagInfo {
+	r.RLock()
+	defer r.RUnlock()
+
+	out := make([]DagInfo, 0, len(r.dagOrder))
+	for _, dagID := range r.dagOrder {
+		taskIDs := r.taskOrder[dagID]
+		tasks := make([]TaskInfo, 0, len(taskIDs))
+		for _, tid := range taskIDs {
+			tasks = append(tasks, TaskInfo{ID: tid})
+		}
+		out = append(out, DagInfo{DagID: dagID, Tasks: tasks})
+	}
+	return out
+}

go-sdk/bundle/bundlev1/registry_test.go

@@ -118,3 +118,38 @@ func (s *RegistrySuite) TestAddTask_ErrorReturnType() {
 	_, exists := s.reg.LookupTask("dag1", "errorTask")
 	s.True(exists)
 }
+
+func (s *RegistrySuite) TestOrderedDags_Empty() {
+	enum, ok := New().(EnumerableBundle)
+	s.Require().True(ok)
+	s.Empty(enum.OrderedDags())
+}
+
+func (s *RegistrySuite) TestOrderedDags_PreservesRegistrationOrder() {
+	reg := New()
+	// Register dags out of alphabetical order so the test fails if OrderedDags
+	// ever sorts instead of preserving AddDag order.
+	zeta := reg.AddDag("zeta")
+	zeta.AddTaskWithName("z1", myTask)
+	zeta.AddTaskWithName("z2", myTask)
+
+	alpha := reg.AddDag("alpha")
+	alpha.AddTaskWithName("a1", myTask)
+
+	reg.AddDag("mid") // dag with no tasks
+
+	enum, ok := reg.(EnumerableBundle)
+	s.Require().True(ok)
+
+	got := enum.OrderedDags()
+	s.Require().Len(got, 3)
+
+	s.Equal("zeta", got[0].DagID)
+	s.Equal([]TaskInfo{{ID: "z1"}, {ID: "z2"}}, got[0].Tasks)
+
+	s.Equal("alpha", got[1].DagID)
+	s.Equal([]TaskInfo{{ID: "a1"}}, got[1].Tasks)
+
+	s.Equal("mid", got[2].DagID)
+	s.Empty(got[2].Tasks)
+}