From 6d12d01727530f0d997c8fd9fbbbb24ca4021d30 Mon Sep 17 00:00:00 2001 From: "Guan-Ming (Wesley) Chiu" <105915352+guan404ming@users.noreply.github.com> Date: Fri, 3 Jul 2026 13:40:55 +0800 Subject: [PATCH] Add airflow-ts-pack bundle build tool for TypeScript SDK --- .../sdk/coordinators/node/coordinator.py | 51 +++- .../coordinators/node/test_coordinator.py | 82 +++-- ts-sdk/README.md | 31 +- ts-sdk/example/README.md | 19 +- ts-sdk/example/package.json | 3 +- ts-sdk/package.json | 6 +- ts-sdk/pnpm-lock.yaml | 289 +++++++++++++++++- ts-sdk/src/cli/main.ts | 28 ++ ts-sdk/src/cli/pack.ts | 166 ++++++++++ ts-sdk/src/coordinator/runtime.ts | 26 +- ts-sdk/tests/cli/fixtures/entry.ts | 26 ++ ts-sdk/tests/cli/pack.test.ts | 137 +++++++++ .../coordinator/runtime-manifest.test.ts | 58 ++++ 13 files changed, 867 insertions(+), 55 deletions(-) create mode 100644 ts-sdk/src/cli/main.ts create mode 100644 ts-sdk/src/cli/pack.ts create mode 100644 ts-sdk/tests/cli/fixtures/entry.ts create mode 100644 ts-sdk/tests/cli/pack.test.ts create mode 100644 ts-sdk/tests/coordinator/runtime-manifest.test.ts diff --git a/task-sdk/src/airflow/sdk/coordinators/node/coordinator.py b/task-sdk/src/airflow/sdk/coordinators/node/coordinator.py index c1b23615b89a1..c065eda0cd0ad 100644 --- a/task-sdk/src/airflow/sdk/coordinators/node/coordinator.py +++ b/task-sdk/src/airflow/sdk/coordinators/node/coordinator.py @@ -19,6 +19,7 @@ from __future__ import annotations +import base64 import os import pathlib from typing import TYPE_CHECKING, Any @@ -41,6 +42,9 @@ BUNDLE_FILENAME = "bundle.mjs" METADATA_FILENAME = "airflow-metadata.yaml" +EMBEDDED_METADATA_MARKER = b"\n//# airflowMetadata=" +# Metadata sits on the bundle's last line, so scanning the trailing 1 MiB is enough. +EMBEDDED_METADATA_TAIL_BYTES = 1 << 20 def _validate_schema_version(instance, _, value) -> str: @@ -53,6 +57,38 @@ class _NodeBundle: schema_version: str = attrs.field(validator=_validate_schema_version) +def _read_embedded_metadata(bundle_path: pathlib.Path) -> dict[str, Any] | None: + """ + Read the manifest ``airflow-ts-pack`` embeds in the bundle itself. + + The packer appends the ``airflow-metadata.yaml`` content as a trailing + ``//# airflowMetadata=`` line comment, keeping bundle and metadata + a single artifact. Returns ``None`` when the bundle has no such marker. + """ + try: + with bundle_path.open("rb") as bundle_file: + bundle_file.seek(0, os.SEEK_END) + size = bundle_file.tell() + bundle_file.seek(max(0, size - EMBEDDED_METADATA_TAIL_BYTES)) + tail = bundle_file.read() + except OSError as exc: + raise ValueError(f"cannot read {bundle_path.name}: {exc}") from exc + + marker_at = tail.rfind(EMBEDDED_METADATA_MARKER) + if marker_at == -1: + return None + + payload = tail[marker_at + len(EMBEDDED_METADATA_MARKER) :].split(b"\n", 1)[0].strip() + try: + data = yaml.safe_load(base64.b64decode(payload, validate=True).decode("utf-8")) + except (ValueError, yaml.YAMLError) as exc: + raise ValueError(f"cannot parse embedded airflow metadata: {exc}") from exc + + if not isinstance(data, dict): + raise ValueError("embedded airflow metadata must contain a mapping") + return data + + def _read_bundle_metadata(metadata_path: pathlib.Path) -> dict[str, Any]: if not metadata_path.is_file(): raise ValueError(f"missing {METADATA_FILENAME}") @@ -85,8 +121,9 @@ def _find_bundle(bundles_root: Sequence[pathlib.Path]) -> _NodeBundle: """ Locate the ``.mjs`` entry point in *bundles_root*. - Scans each configured directory for ``bundle.mjs`` and reads the sibling - ``airflow-metadata.yaml`` for the bundle's supervisor schema version. + Scans each configured directory for ``bundle.mjs`` and reads the bundle's + supervisor schema version from the metadata embedded in the bundle, + falling back to a sibling ``airflow-metadata.yaml`` sidecar. This is an ordered fallback search, not Dag/task-aware multi-bundle routing. The first bundle found wins. A future version can use the @@ -99,7 +136,9 @@ def _find_bundle(bundles_root: Sequence[pathlib.Path]) -> _NodeBundle: if not candidate.is_file(): continue try: - metadata = _read_bundle_metadata(root / METADATA_FILENAME) + metadata = _read_embedded_metadata(candidate) + if metadata is None: + metadata = _read_bundle_metadata(root / METADATA_FILENAME) log.debug("Selected TypeScript bundle", path=candidate, root=root) return _NodeBundle( path=candidate, @@ -155,8 +194,10 @@ class NodeCoordinator(SubprocessCoordinator): ``"node"``, which relies on ``$PATH``). :param bundles_root: Ordered list of directories scanned for a usable TypeScript bundle. Each bundle directory must contain ``bundle.mjs`` - and ``airflow-metadata.yaml``. This is a fallback search path; it does - not yet route different Dag/task pairs to different bundles. + with embedded metadata (as produced by ``airflow-ts-pack``), or + ``bundle.mjs`` plus an ``airflow-metadata.yaml`` sidecar. This is a + fallback search path; it does not yet route different Dag/task pairs + to different bundles. :param task_startup_timeout: Maximum time the coordinator waits for a task process to start, in seconds. The default is 10 seconds. """ diff --git a/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py b/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py index 3538c389a19b2..919698fa6cfe8 100644 --- a/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py +++ b/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py @@ -18,6 +18,7 @@ from __future__ import annotations +import base64 import pathlib import pytest @@ -45,27 +46,36 @@ def _make_ti(dag_id: str = "test_dag", queue: str = "ts") -> TaskInstance: ) +def _metadata_yaml(schema_version: str) -> str: + return "\n".join( + [ + 'airflow_bundle_metadata_version: "1.0"', + "sdk:", + " language: typescript", + ' version: "0.1.0"', + f' supervisor_schema_version: "{schema_version}"', + "source: src/airflow.ts", + "dags:", + " test_dag:", + " tasks:", + " - test_task", + "", + ] + ) + + def write_bundle(root: pathlib.Path, schema_version: str = SCHEMA_VERSION) -> pathlib.Path: bundle = root / "bundle.mjs" bundle.write_text("export {};\n", encoding="utf-8") - (root / "airflow-metadata.yaml").write_text( - "\n".join( - [ - 'airflow_bundle_metadata_version: "1.0"', - "sdk:", - " language: typescript", - ' version: "0.1.0"', - f' supervisor_schema_version: "{schema_version}"', - "source: src/airflow.ts", - "dags:", - " test_dag:", - " tasks:", - " - test_task", - "", - ] - ), - encoding="utf-8", - ) + (root / "airflow-metadata.yaml").write_text(_metadata_yaml(schema_version), encoding="utf-8") + return bundle + + +def write_embedded_bundle(root: pathlib.Path, payload: str | None = None) -> pathlib.Path: + if payload is None: + payload = base64.b64encode(_metadata_yaml(SCHEMA_VERSION).encode("utf-8")).decode("ascii") + bundle = root / "bundle.mjs" + bundle.write_text(f"export {{}};\n//# airflowMetadata={payload}\n", encoding="utf-8") return bundle @@ -105,6 +115,42 @@ def test_find_bundle_returns_bundle_mjs(self, tmp_path): assert found.path == bundle assert found.schema_version == SCHEMA_VERSION + def test_find_bundle_reads_embedded_metadata_without_sidecar(self, tmp_path): + bundle = write_embedded_bundle(tmp_path) + + found = _find_bundle([tmp_path]) + + assert found.path == bundle + assert found.schema_version == SCHEMA_VERSION + + def test_find_bundle_prefers_embedded_metadata_over_sidecar(self, tmp_path): + bundle = write_embedded_bundle(tmp_path) + (tmp_path / "airflow-metadata.yaml").write_text("[not-a-mapping]\n", encoding="utf-8") + + found = _find_bundle([tmp_path]) + + assert found.path == bundle + assert found.schema_version == SCHEMA_VERSION + + @pytest.mark.parametrize( + ("payload", "message"), + [ + ("not-base64!", "cannot parse embedded airflow metadata"), + (base64.b64encode(b"[not-a-mapping]").decode("ascii"), "must contain a mapping"), + ], + ) + def test_find_bundle_rejects_invalid_embedded_metadata(self, tmp_path, payload, message): + write_embedded_bundle(tmp_path, payload=payload) + + with pytest.raises(FileNotFoundError, match=message): + _find_bundle([tmp_path]) + + def test_find_bundle_rejects_empty_marker_at_end_of_file(self, tmp_path): + (tmp_path / "bundle.mjs").write_text("export {};\n//# airflowMetadata=", encoding="utf-8") + + with pytest.raises(FileNotFoundError, match="must contain a mapping"): + _find_bundle([tmp_path]) + def test_find_bundle_checks_multiple_roots(self, tmp_path): first = tmp_path / "first" second = tmp_path / "second" diff --git a/ts-sdk/README.md b/ts-sdk/README.md index 3ac7360aca609..ad9b6d54787d2 100644 --- a/ts-sdk/README.md +++ b/ts-sdk/README.md @@ -90,8 +90,10 @@ coordinators = { queue_to_coordinator = {"typescript": "ts"} ``` -Each configured bundle directory must contain `bundle.mjs` and -`airflow-metadata.yaml`. +Each configured bundle directory must contain a `bundle.mjs` built with +`airflow-ts-pack` (see [Packing bundles](#packing-bundles)), which embeds the +Airflow metadata in the bundle itself. A `bundle.mjs` without embedded +metadata is also accepted alongside an `airflow-metadata.yaml` sidecar. TypeScript entrypoint: @@ -147,8 +149,29 @@ Airflow launches the bundled entrypoint with `--comm=host:port` and the task startup message, finds the registered handler for the Dag/task pair, and reports the terminal task state back to Airflow. -See [`example/`](example/) for a coordinator-runtime example that builds a -`bundle.mjs` with `esbuild` and uses a Python stub Dag. +See [`example/`](example/) for a coordinator-runtime example that packs a +bundle with `airflow-ts-pack` and uses a Python stub Dag. + +## Packing bundles + +`airflow-ts-pack` produces everything `NodeCoordinator` needs in one command: + +```bash +airflow-ts-pack src/main.ts --outdir dist +``` + +It bundles the entrypoint into `dist/bundle.mjs` with esbuild, runs the +bundle with `--airflow-metadata` so the bundle reports its own registered +Dag/task pairs and supervisor schema version, and embeds that manifest in the +bundle as a trailing `//# airflowMetadata=` comment. The result is a +single deployable file whose metadata cannot drift from its code; no +hand-written sidecar is needed. + +Options: + +- `--outdir ` — output directory (default `dist`) +- `--source ` — display name of the primary source file shown in the + Airflow UI (default: entry basename) ## TaskClient diff --git a/ts-sdk/example/README.md b/ts-sdk/example/README.md index 181135a52312c..1ea267bdedf83 100644 --- a/ts-sdk/example/README.md +++ b/ts-sdk/example/README.md @@ -26,8 +26,9 @@ This example shows the coordinator-mode shape for TypeScript task handlers: starts the coordinator runtime. - `dist/bundle.mjs` is the generated Node.js bundle that Airflow launches. -The TypeScript SDK does not include a packer yet, so this example builds the -bundle with `esbuild` and writes the Airflow metadata file manually. +The build uses the SDK's `airflow-ts-pack` tool, which bundles the entrypoint +with esbuild and embeds the Airflow metadata generated from the bundle's +registered tasks, producing a single deployable file. ## Build @@ -39,7 +40,7 @@ pnpm install pnpm run build ``` -Build the example bundle: +Build the example bundle and its metadata: ```bash cd ts-sdk/example @@ -47,23 +48,11 @@ pnpm install pnpm run build ``` -Create the metadata file next to the generated bundle: - -```bash -node --input-type=module > dist/airflow-metadata.yaml <<'EOF' -import { SUPERVISOR_API_VERSION } from "@apache-airflow/ts-sdk"; - -console.log(`sdk: - supervisor_schema_version: "${SUPERVISOR_API_VERSION}"`); -EOF -``` - The coordinator expects this layout: ```text ts-sdk/example/dist/ bundle.mjs - airflow-metadata.yaml ``` ## Airflow Configuration diff --git a/ts-sdk/example/package.json b/ts-sdk/example/package.json index 5594727113c13..f7daf4901cc3e 100644 --- a/ts-sdk/example/package.json +++ b/ts-sdk/example/package.json @@ -5,7 +5,7 @@ "type": "module", "license": "Apache-2.0", "scripts": { - "build": "esbuild src/main.ts --bundle --platform=node --format=esm --target=node22 --outfile=dist/bundle.mjs", + "build": "airflow-ts-pack src/main.ts --outdir dist", "typecheck": "tsc --noEmit" }, "dependencies": { @@ -13,7 +13,6 @@ }, "devDependencies": { "@types/node": "^22.19.17", - "esbuild": "^0.28.1", "typescript": "^6.0.2" } } diff --git a/ts-sdk/package.json b/ts-sdk/package.json index 190df17101456..ef092dd13227c 100644 --- a/ts-sdk/package.json +++ b/ts-sdk/package.json @@ -7,6 +7,9 @@ "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", + "bin": { + "airflow-ts-pack": "./dist/cli/main.js" + }, "homepage": "https://airflow.apache.org", "repository": { "type": "git", @@ -57,7 +60,8 @@ "node": ">=22" }, "dependencies": { - "@msgpack/msgpack": "^3.1.2" + "@msgpack/msgpack": "^3.1.2", + "esbuild": "^0.28.1" }, "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/ts-sdk/pnpm-lock.yaml b/ts-sdk/pnpm-lock.yaml index 887acdcc7e418..1bf8a34ecc92e 100644 --- a/ts-sdk/pnpm-lock.yaml +++ b/ts-sdk/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@msgpack/msgpack': specifier: ^3.1.2 version: 3.1.3 + esbuild: + specifier: ^0.28.1 + version: 0.28.1 devDependencies: '@eslint/js': specifier: ^10.0.1 @@ -38,7 +41,7 @@ importers: version: 8.60.0(eslint@10.4.0)(typescript@6.0.3) vitest: specifier: ^4.1.7 - version: 4.1.7(@types/node@22.19.17)(@vitest/coverage-v8@4.1.7)(vite@8.0.10(@types/node@22.19.17)(esbuild@0.27.7)(tsx@4.21.0)) + version: 4.1.7(@types/node@22.19.17)(@vitest/coverage-v8@4.1.7)(vite@8.0.10(@types/node@22.19.17)(esbuild@0.28.1)(tsx@4.21.0)) packages: @@ -82,156 +85,312 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/android-arm64@0.27.7': resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} engines: {node: '>=18'} cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm@0.27.7': resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} engines: {node: '>=18'} cpu: [arm] os: [android] + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-x64@0.27.7': resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} engines: {node: '>=18'} cpu: [x64] os: [android] + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/darwin-arm64@0.27.7': resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-x64@0.27.7': resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/freebsd-arm64@0.27.7': resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-x64@0.27.7': resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/linux-arm64@0.27.7': resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} engines: {node: '>=18'} cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm@0.27.7': resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} engines: {node: '>=18'} cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-ia32@0.27.7': resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} engines: {node: '>=18'} cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-loong64@0.27.7': resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} engines: {node: '>=18'} cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-mips64el@0.27.7': resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-ppc64@0.27.7': resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-riscv64@0.27.7': resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-s390x@0.27.7': resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} engines: {node: '>=18'} cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-x64@0.27.7': resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} engines: {node: '>=18'} cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/netbsd-arm64@0.27.7': resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-x64@0.27.7': resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/openbsd-arm64@0.27.7': resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-x64@0.27.7': resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openharmony-arm64@0.27.7': resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + '@esbuild/sunos-x64@0.27.7': resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} engines: {node: '>=18'} cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/win32-arm64@0.27.7': resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-ia32@0.27.7': resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} engines: {node: '>=18'} cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-x64@0.27.7': resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} engines: {node: '>=18'} cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@eslint-community/eslint-utils@4.9.1': resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -599,6 +758,11 @@ packages: engines: {node: '>=18'} hasBin: true + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -1160,81 +1324,159 @@ snapshots: '@esbuild/aix-ppc64@0.27.7': optional: true + '@esbuild/aix-ppc64@0.28.1': + optional: true + '@esbuild/android-arm64@0.27.7': optional: true + '@esbuild/android-arm64@0.28.1': + optional: true + '@esbuild/android-arm@0.27.7': optional: true + '@esbuild/android-arm@0.28.1': + optional: true + '@esbuild/android-x64@0.27.7': optional: true + '@esbuild/android-x64@0.28.1': + optional: true + '@esbuild/darwin-arm64@0.27.7': optional: true + '@esbuild/darwin-arm64@0.28.1': + optional: true + '@esbuild/darwin-x64@0.27.7': optional: true + '@esbuild/darwin-x64@0.28.1': + optional: true + '@esbuild/freebsd-arm64@0.27.7': optional: true + '@esbuild/freebsd-arm64@0.28.1': + optional: true + '@esbuild/freebsd-x64@0.27.7': optional: true + '@esbuild/freebsd-x64@0.28.1': + optional: true + '@esbuild/linux-arm64@0.27.7': optional: true + '@esbuild/linux-arm64@0.28.1': + optional: true + '@esbuild/linux-arm@0.27.7': optional: true + '@esbuild/linux-arm@0.28.1': + optional: true + '@esbuild/linux-ia32@0.27.7': optional: true + '@esbuild/linux-ia32@0.28.1': + optional: true + '@esbuild/linux-loong64@0.27.7': optional: true + '@esbuild/linux-loong64@0.28.1': + optional: true + '@esbuild/linux-mips64el@0.27.7': optional: true + '@esbuild/linux-mips64el@0.28.1': + optional: true + '@esbuild/linux-ppc64@0.27.7': optional: true + '@esbuild/linux-ppc64@0.28.1': + optional: true + '@esbuild/linux-riscv64@0.27.7': optional: true + '@esbuild/linux-riscv64@0.28.1': + optional: true + '@esbuild/linux-s390x@0.27.7': optional: true + '@esbuild/linux-s390x@0.28.1': + optional: true + '@esbuild/linux-x64@0.27.7': optional: true + '@esbuild/linux-x64@0.28.1': + optional: true + '@esbuild/netbsd-arm64@0.27.7': optional: true + '@esbuild/netbsd-arm64@0.28.1': + optional: true + '@esbuild/netbsd-x64@0.27.7': optional: true + '@esbuild/netbsd-x64@0.28.1': + optional: true + '@esbuild/openbsd-arm64@0.27.7': optional: true + '@esbuild/openbsd-arm64@0.28.1': + optional: true + '@esbuild/openbsd-x64@0.27.7': optional: true + '@esbuild/openbsd-x64@0.28.1': + optional: true + '@esbuild/openharmony-arm64@0.27.7': optional: true + '@esbuild/openharmony-arm64@0.28.1': + optional: true + '@esbuild/sunos-x64@0.27.7': optional: true + '@esbuild/sunos-x64@0.28.1': + optional: true + '@esbuild/win32-arm64@0.27.7': optional: true + '@esbuild/win32-arm64@0.28.1': + optional: true + '@esbuild/win32-ia32@0.27.7': optional: true + '@esbuild/win32-ia32@0.28.1': + optional: true + '@esbuild/win32-x64@0.27.7': optional: true + '@esbuild/win32-x64@0.28.1': + optional: true + '@eslint-community/eslint-utils@4.9.1(eslint@10.4.0)': dependencies: eslint: 10.4.0 @@ -1489,7 +1731,7 @@ snapshots: obug: 2.1.1 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.7(@types/node@22.19.17)(@vitest/coverage-v8@4.1.7)(vite@8.0.10(@types/node@22.19.17)(esbuild@0.27.7)(tsx@4.21.0)) + vitest: 4.1.7(@types/node@22.19.17)(@vitest/coverage-v8@4.1.7)(vite@8.0.10(@types/node@22.19.17)(esbuild@0.28.1)(tsx@4.21.0)) optional: true '@vitest/expect@4.1.7': @@ -1501,13 +1743,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.7(vite@8.0.10(@types/node@22.19.17)(esbuild@0.27.7)(tsx@4.21.0))': + '@vitest/mocker@4.1.7(vite@8.0.10(@types/node@22.19.17)(esbuild@0.28.1)(tsx@4.21.0))': dependencies: '@vitest/spy': 4.1.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.10(@types/node@22.19.17)(esbuild@0.27.7)(tsx@4.21.0) + vite: 8.0.10(@types/node@22.19.17)(esbuild@0.28.1)(tsx@4.21.0) '@vitest/pretty-format@4.1.7': dependencies: @@ -1612,6 +1854,35 @@ snapshots: '@esbuild/win32-ia32': 0.27.7 '@esbuild/win32-x64': 0.27.7 + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + escape-string-regexp@4.0.0: {} eslint-scope@9.1.2: @@ -2007,7 +2278,7 @@ snapshots: dependencies: punycode: 2.3.1 - vite@8.0.10(@types/node@22.19.17)(esbuild@0.27.7)(tsx@4.21.0): + vite@8.0.10(@types/node@22.19.17)(esbuild@0.28.1)(tsx@4.21.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -2016,14 +2287,14 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 22.19.17 - esbuild: 0.27.7 + esbuild: 0.28.1 fsevents: 2.3.3 tsx: 4.21.0 - vitest@4.1.7(@types/node@22.19.17)(@vitest/coverage-v8@4.1.7)(vite@8.0.10(@types/node@22.19.17)(esbuild@0.27.7)(tsx@4.21.0)): + vitest@4.1.7(@types/node@22.19.17)(@vitest/coverage-v8@4.1.7)(vite@8.0.10(@types/node@22.19.17)(esbuild@0.28.1)(tsx@4.21.0)): dependencies: '@vitest/expect': 4.1.7 - '@vitest/mocker': 4.1.7(vite@8.0.10(@types/node@22.19.17)(esbuild@0.27.7)(tsx@4.21.0)) + '@vitest/mocker': 4.1.7(vite@8.0.10(@types/node@22.19.17)(esbuild@0.28.1)(tsx@4.21.0)) '@vitest/pretty-format': 4.1.7 '@vitest/runner': 4.1.7 '@vitest/snapshot': 4.1.7 @@ -2040,7 +2311,7 @@ snapshots: tinyexec: 1.1.1 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.10(@types/node@22.19.17)(esbuild@0.27.7)(tsx@4.21.0) + vite: 8.0.10(@types/node@22.19.17)(esbuild@0.28.1)(tsx@4.21.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 22.19.17 diff --git a/ts-sdk/src/cli/main.ts b/ts-sdk/src/cli/main.ts new file mode 100644 index 0000000000000..1db958183fe20 --- /dev/null +++ b/ts-sdk/src/cli/main.ts @@ -0,0 +1,28 @@ +#!/usr/bin/env node +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { runPack } from "./pack.js"; + +try { + await runPack(process.argv.slice(2)); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +} diff --git a/ts-sdk/src/cli/pack.ts b/ts-sdk/src/cli/pack.ts new file mode 100644 index 0000000000000..a76ddc3132ee1 --- /dev/null +++ b/ts-sdk/src/cli/pack.ts @@ -0,0 +1,166 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// airflow-ts-pack: bundle a TypeScript entrypoint into the single-file +// artifact NodeCoordinator consumes — `bundle.mjs` with the airflow +// metadata embedded as a trailing `//# airflowMetadata=` comment. +// +// Mirrors airflow-go-pack: build first, then run the built bundle with +// --airflow-metadata so the manifest comes from the bundle's own task +// registry and schema version, never from a hand-written sidecar. + +import { execFileSync } from "node:child_process"; +import { appendFileSync, readFileSync } from "node:fs"; +import path from "node:path"; + +import { build } from "esbuild"; + +import { AIRFLOW_METADATA_FLAG, type BundleManifest } from "../coordinator/runtime.js"; + +const AIRFLOW_BUNDLE_METADATA_VERSION = "1.0"; +const BUNDLE_FILENAME = "bundle.mjs"; +export const EMBEDDED_METADATA_PREFIX = "//# airflowMetadata="; + +const USAGE = `Usage: airflow-ts-pack [--outdir ] [--source ] + +Bundles into /${BUNDLE_FILENAME} with esbuild and embeds the +airflow metadata generated from the bundle's registered tasks. + +Options: + --outdir Output directory (default: dist) + --source Display name of the primary source file (default: basename) +`; + +export interface PackArgs { + entry: string; + outdir: string; + source: string; +} + +export function parsePackArgs(argv: readonly string[]): PackArgs { + let entry: string | null = null; + let outdir = "dist"; + let source: string | null = null; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]!; + if (arg === "--outdir" || arg === "--source") { + const value = argv[i + 1]; + if (!value) throw new Error(`${arg} requires a value\n\n${USAGE}`); + if (arg === "--outdir") outdir = value; + else source = value; + i += 1; + } else if (arg.startsWith("-")) { + throw new Error(`Unknown option ${arg}\n\n${USAGE}`); + } else if (entry) { + throw new Error(`Unexpected argument ${arg}\n\n${USAGE}`); + } else { + entry = arg; + } + } + if (!entry) throw new Error(`Missing entry file\n\n${USAGE}`); + return { entry, outdir, source: source ?? path.basename(entry) }; +} + +export interface PackMetadata { + airflow_bundle_metadata_version: string; + sdk: { language: string; version: string; supervisor_schema_version: string }; + source: string; + dags: BundleManifest["dags"]; +} + +// JSON string literals are valid YAML double-quoted scalars, so every +// scalar below is emitted through JSON.stringify for correct escaping. +export function renderMetadataYaml(metadata: PackMetadata): string { + const lines = [ + `airflow_bundle_metadata_version: ${JSON.stringify(metadata.airflow_bundle_metadata_version)}`, + "sdk:", + ` language: ${JSON.stringify(metadata.sdk.language)}`, + ` version: ${JSON.stringify(metadata.sdk.version)}`, + ` supervisor_schema_version: ${JSON.stringify(metadata.sdk.supervisor_schema_version)}`, + `source: ${JSON.stringify(metadata.source)}`, + "dags:", + ]; + for (const [dagId, dag] of Object.entries(metadata.dags)) { + lines.push(` ${JSON.stringify(dagId)}:`); + lines.push(` tasks: [${dag.tasks.map((task) => JSON.stringify(task)).join(", ")}]`); + } + return `${lines.join("\n")}\n`; +} + +function readSdkVersion(): string { + const packageJsonUrl = new URL("../../package.json", import.meta.url); + const { version } = JSON.parse(readFileSync(packageJsonUrl, "utf-8")) as { version: string }; + return version; +} + +function readBundleManifest(bundlePath: string): BundleManifest { + const stdout = execFileSync(process.execPath, [bundlePath, AIRFLOW_METADATA_FLAG], { + encoding: "utf-8", + }); + let manifest: BundleManifest; + try { + manifest = JSON.parse(stdout) as BundleManifest; + } catch (error) { + throw new Error(`Bundle produced invalid --airflow-metadata output: ${String(error)}`, { + cause: error, + }); + } + if (!manifest.supervisor_schema_version || typeof manifest.dags !== "object") { + throw new Error("Bundle produced incomplete --airflow-metadata output"); + } + return manifest; +} + +export async function runPack(argv: readonly string[]): Promise { + const args = parsePackArgs(argv); + const bundlePath = path.join(args.outdir, BUNDLE_FILENAME); + + await build({ + entryPoints: [args.entry], + bundle: true, + platform: "node", + format: "esm", + target: "node22", + outfile: bundlePath, + }); + + const manifest = readBundleManifest(bundlePath); + if (Object.keys(manifest.dags).length === 0) { + throw new Error( + `${args.entry} registered no tasks; call registerTask(...) before startCoordinator()`, + ); + } + + const metadataYaml = renderMetadataYaml({ + airflow_bundle_metadata_version: AIRFLOW_BUNDLE_METADATA_VERSION, + sdk: { + language: "typescript", + version: readSdkVersion(), + supervisor_schema_version: manifest.supervisor_schema_version, + }, + source: args.source, + dags: manifest.dags, + }); + appendFileSync( + bundlePath, + `\n${EMBEDDED_METADATA_PREFIX}${Buffer.from(metadataYaml, "utf-8").toString("base64")}\n`, + ); + + console.log(`Wrote ${bundlePath} (airflow metadata embedded)`); +} diff --git a/ts-sdk/src/coordinator/runtime.ts b/ts-sdk/src/coordinator/runtime.ts index d083bf4acce85..96b73b962ea29 100644 --- a/ts-sdk/src/coordinator/runtime.ts +++ b/ts-sdk/src/coordinator/runtime.ts @@ -46,7 +46,7 @@ import { type RuntimeTaskState, type StartupDetails, } from "./protocol.js"; -import { getRegisteredTask, listRegisteredTasks } from "../sdk/registry.js"; +import { getRegisteredTask, listRegisteredTasks, type TaskRegistration } from "../sdk/registry.js"; import type { TaskContext, TaskHandlerArgs } from "../sdk/task.js"; import type { JsonValue } from "../sdk/client-types.js"; @@ -103,10 +103,34 @@ export function parseArgs(argv: readonly string[]): ParsedArgs { return { commAddr, logsAddr }; } +export const AIRFLOW_METADATA_FLAG = "--airflow-metadata"; + +/** Bundle manifest fields only the built bundle itself knows: the schema + * version it was compiled against and the Dag/task pairs it registered. + * `airflow-ts-pack` runs `node bundle.mjs --airflow-metadata` to read this. */ +export interface BundleManifest { + supervisor_schema_version: string; + dags: Record; +} + +export function buildBundleManifest( + registrations: readonly TaskRegistration[] = listRegisteredTasks(), +): BundleManifest { + const dags: BundleManifest["dags"] = {}; + for (const { dagId, taskId } of registrations) { + (dags[dagId] ??= { tasks: [] }).tasks.push(taskId); + } + return { supervisor_schema_version: SUPERVISOR_API_VERSION, dags }; +} + /** Start the coordinator runtime. Resolves when the subprocess has * delivered its terminal frame and closed both sockets. */ export async function startCoordinator(opts: StartCoordinatorOptions = {}): Promise { const argv = opts.argv ?? process.argv; + if (argv.includes(AIRFLOW_METADATA_FLAG)) { + process.stdout.write(`${JSON.stringify(buildBundleManifest())}\n`); + return; + } const parsed = opts.commAddr && opts.logsAddr ? { commAddr: opts.commAddr, logsAddr: opts.logsAddr } diff --git a/ts-sdk/tests/cli/fixtures/entry.ts b/ts-sdk/tests/cli/fixtures/entry.ts new file mode 100644 index 0000000000000..8d53c43aee500 --- /dev/null +++ b/ts-sdk/tests/cli/fixtures/entry.ts @@ -0,0 +1,26 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { registerTask, startCoordinator } from "../../../src/index.js"; + +registerTask({ dagId: "fixture_dag", taskId: "extract" }, async () => "extracted"); +registerTask({ dagId: "fixture_dag", taskId: "transform" }, async () => "transformed"); +registerTask({ dagId: "other_dag", taskId: "solo" }, async () => undefined); + +await startCoordinator(); diff --git a/ts-sdk/tests/cli/pack.test.ts b/ts-sdk/tests/cli/pack.test.ts new file mode 100644 index 0000000000000..9e6187fea3273 --- /dev/null +++ b/ts-sdk/tests/cli/pack.test.ts @@ -0,0 +1,137 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + EMBEDDED_METADATA_PREFIX, + parsePackArgs, + renderMetadataYaml, + runPack, +} from "../../src/cli/pack.js"; +import { SUPERVISOR_API_VERSION } from "../../src/coordinator/protocol.js"; + +const FIXTURE_ENTRY = fileURLToPath(new URL("fixtures/entry.ts", import.meta.url)); +const SDK_VERSION = ( + JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf-8")) as { + version: string; + } +).version; + +describe("parsePackArgs", () => { + it("parses entry with defaults", () => { + expect(parsePackArgs(["src/main.ts"])).toEqual({ + entry: "src/main.ts", + outdir: "dist", + source: "main.ts", + }); + }); + + it("parses --outdir and --source overrides", () => { + expect(parsePackArgs(["src/main.ts", "--outdir", "build", "--source", "pipeline.ts"])).toEqual({ + entry: "src/main.ts", + outdir: "build", + source: "pipeline.ts", + }); + }); + + it.each([ + [[], "Missing entry file"], + [["--outdir"], "--outdir requires a value"], + [["a.ts", "b.ts"], "Unexpected argument b.ts"], + [["a.ts", "--bogus"], "Unknown option --bogus"], + ])("rejects %j", (argv, message) => { + expect(() => parsePackArgs(argv)).toThrow(message); + }); +}); + +describe("renderMetadataYaml", () => { + it("emits schema-conformant YAML with escaped scalars", () => { + const yaml = renderMetadataYaml({ + airflow_bundle_metadata_version: "1.0", + sdk: { language: "typescript", version: "0.1.0", supervisor_schema_version: "2026-06-16" }, + source: 'we"ird.ts', + dags: { my_dag: { tasks: ["a", 'b"c'] } }, + }); + expect(yaml).toBe( + [ + 'airflow_bundle_metadata_version: "1.0"', + "sdk:", + ' language: "typescript"', + ' version: "0.1.0"', + ' supervisor_schema_version: "2026-06-16"', + 'source: "we\\"ird.ts"', + "dags:", + ' "my_dag":', + ' tasks: ["a", "b\\"c"]', + "", + ].join("\n"), + ); + }); +}); + +describe("runPack", () => { + let outdir: string; + + afterEach(() => { + if (outdir) rmSync(outdir, { recursive: true, force: true }); + }); + + it("bundles the entry and embeds metadata from the bundle's registry", async () => { + outdir = mkdtempSync(path.join(tmpdir(), "ts-pack-")); + const nested = path.join(outdir, "dist"); + await runPack([FIXTURE_ENTRY, "--outdir", nested]); + + const bundlePath = path.join(nested, "bundle.mjs"); + expect(existsSync(path.join(nested, "airflow-metadata.yaml"))).toBe(false); + + const lastLine = readFileSync(bundlePath, "utf-8").trimEnd().split("\n").at(-1)!; + expect(lastLine.startsWith(EMBEDDED_METADATA_PREFIX)).toBe(true); + const metadata = Buffer.from( + lastLine.slice(EMBEDDED_METADATA_PREFIX.length), + "base64", + ).toString("utf-8"); + expect(metadata).toBe( + [ + 'airflow_bundle_metadata_version: "1.0"', + "sdk:", + ' language: "typescript"', + ` version: ${JSON.stringify(SDK_VERSION)}`, + ` supervisor_schema_version: ${JSON.stringify(SUPERVISOR_API_VERSION)}`, + 'source: "entry.ts"', + "dags:", + ' "fixture_dag":', + ' tasks: ["extract", "transform"]', + ' "other_dag":', + ' tasks: ["solo"]', + "", + ].join("\n"), + ); + + const dumped = execFileSync(process.execPath, [bundlePath, "--airflow-metadata"], { + encoding: "utf-8", + }); + expect(JSON.parse(dumped).supervisor_schema_version).toBe(SUPERVISOR_API_VERSION); + }); +}); diff --git a/ts-sdk/tests/coordinator/runtime-manifest.test.ts b/ts-sdk/tests/coordinator/runtime-manifest.test.ts new file mode 100644 index 0000000000000..f5b919fe499c7 --- /dev/null +++ b/ts-sdk/tests/coordinator/runtime-manifest.test.ts @@ -0,0 +1,58 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { buildBundleManifest, startCoordinator } from "../../src/coordinator/runtime.js"; +import { SUPERVISOR_API_VERSION } from "../../src/coordinator/protocol.js"; + +describe("buildBundleManifest", () => { + it("groups registrations by Dag ID under the SDK's schema version", () => { + expect( + buildBundleManifest([ + { dagId: "dag_a", taskId: "t1" }, + { dagId: "dag_b", taskId: "t2" }, + { dagId: "dag_a", taskId: "t3" }, + ]), + ).toEqual({ + supervisor_schema_version: SUPERVISOR_API_VERSION, + dags: { + dag_a: { tasks: ["t1", "t3"] }, + dag_b: { tasks: ["t2"] }, + }, + }); + }); +}); + +describe("startCoordinator --airflow-metadata", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("dumps the manifest to stdout and returns without connecting", async () => { + const write = vi.spyOn(process.stdout, "write").mockReturnValue(true); + + await startCoordinator({ argv: ["node", "bundle.mjs", "--airflow-metadata"] }); + + expect(write).toHaveBeenCalledTimes(1); + const payload = JSON.parse(String(write.mock.calls[0]![0])); + expect(payload.supervisor_schema_version).toBe(SUPERVISOR_API_VERSION); + expect(payload.dags).toEqual({}); + }); +});