From 0f0d16e4ffb5407119e89698483c7f5cbda61192 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Tue, 21 Jul 2026 16:41:15 -0300 Subject: [PATCH 1/5] W-23530974 test(native-lib): TCK harness core (case loader + comparison) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First of three commits for the Group C conformance harness. Pure, dwlib-free logic that lands in the unit lane; the suite sourcing + driver and CI wiring follow in later commits. - tests/tck/formats.ts: extension ↔ MIME mapping limited to the formats this dwlib compiles in (json, xml, csv, txt, dw, octet-stream, properties, urlencoded, multipart). YAML is not compiled in, so yaml/yml cases are treated as unsupported. - tests/tck/case-loader.ts: parseCase() turns a case directory's file listing into runnable scenarios or a skip decision, mirroring the CLI's structural filters (single transform.dwl, no *-config.properties, no groovy/java, no bare config.properties, no _wip) and dropping unsupported-format scenarios. - tests/tck/compare.ts: compareOutput() does extension-dispatched semantic comparison ported from AssertionHelper — structural JSON deep-equal, whitespace-collapsed XML, EOL-normalized csv/txt/yaml, all-whitespace-stripped dwl, exact bytes for bin. Not naive byte equality. - 33 unit tests (tck-case-loader.test.ts, tck-compare.test.ts). Unit lane: 70 -> 103 tests passing. tsc clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- native-lib/node/tests/tck/case-loader.ts | 138 ++++++++++++++++++ native-lib/node/tests/tck/compare.ts | 123 ++++++++++++++++ native-lib/node/tests/tck/formats.ts | 36 +++++ .../node/tests/unit/tck-case-loader.test.ts | 108 ++++++++++++++ .../node/tests/unit/tck-compare.test.ts | 86 +++++++++++ 5 files changed, 491 insertions(+) create mode 100644 native-lib/node/tests/tck/case-loader.ts create mode 100644 native-lib/node/tests/tck/compare.ts create mode 100644 native-lib/node/tests/tck/formats.ts create mode 100644 native-lib/node/tests/unit/tck-case-loader.test.ts create mode 100644 native-lib/node/tests/unit/tck-compare.test.ts diff --git a/native-lib/node/tests/tck/case-loader.ts b/native-lib/node/tests/tck/case-loader.ts new file mode 100644 index 0000000..509f6e2 --- /dev/null +++ b/native-lib/node/tests/tck/case-loader.ts @@ -0,0 +1,138 @@ +// Parses a TCK case directory into runnable scenarios, applying the same +// structural skip filters as the CLI's TCKCliTest. +// +// TCK case layout (one directory per case, defined by the runtime's +// FolderBasedTest): +// transform.dwl the transform (fixed main file name) +// inN. inputs; basename (in0, in1…) is the variable name, +// extension selects the reader format +// out. expected output; one scenario per out.* file +// config.properties optional per-case config +// +// The parsing here is pure — it operates on a provided list of file names, not +// the filesystem — so it is unit-testable without real TCK fixtures. +import { isSupportedExtension, mimeForExtension } from "./formats"; + +/** Fixed name of the transform script in a TCK case (per FolderBasedTest). */ +export const MAIN_TRANSFORM = "transform.dwl"; + +const INPUT_PATTERN = /^in[0-9]+\.[a-zA-Z]+$/; +const OUTPUT_PATTERN = /^out\.[a-zA-Z]+$/; +const INPUT_CONFIG_PATTERN = /^in[0-9]+-config\.properties$/; +const OUTPUT_CONFIG_PATTERN = /^out[0-9]*-config\.properties$/; + +/** Returns the extension (without dot, lowercased) of a file name, or "" if none. */ +export function extensionOf(name: string): string { + const i = name.lastIndexOf("."); + return i < 0 ? "" : name.slice(i + 1).toLowerCase(); +} + +/** An input file within a case: the DataWeave variable name and its source file. */ +export interface TckInput { + /** Variable name the input binds to (the file's base name, e.g. `in0`). */ + name: string; + /** The input file name (e.g. `in0.json`). */ + fileName: string; + /** MIME type derived from the extension. */ + mimeType: string; +} + +/** One runnable scenario: the transform plus its inputs and a single expected output. */ +export interface TckScenario { + /** Scenario id: `-`. */ + name: string; + inputs: TckInput[]; + /** Expected-output file name (e.g. `out.json`). */ + outputFileName: string; + /** Comparison format, from the output extension. */ + outputMime: string; + outputExtension: string; +} + +/** The result of inspecting one case directory. */ +export type CaseParseResult = + | { kind: "scenarios"; scenarios: TckScenario[] } + | { kind: "skipped"; reason: string }; + +/** + * Parses a case directory (given its file listing) into scenarios, or a skip + * decision. Mirrors the CLI's structural filters: a case is skipped unless it + * has exactly one transform.dwl, and is skipped outright if it carries + * per-input/output config properties, groovy/java cases, a bare + * config.properties, or a _wip marker. Scenarios whose input or output + * extension maps to an unsupported format (e.g. yaml) are dropped; if none + * remain the case is skipped. + * + * @param caseName - The directory (case) name, used as the scenario prefix. + * @param fileNames - The file names directly inside the case directory. + * @returns Either the runnable scenarios or a reason the case was skipped. + */ +export function parseCase(caseName: string, fileNames: string[]): CaseParseResult { + if (caseName.endsWith("_wip") || caseName.endsWith("wip")) { + return { kind: "skipped", reason: "work-in-progress (_wip)" }; + } + + if (fileNames.some((n) => INPUT_CONFIG_PATTERN.test(n) || OUTPUT_CONFIG_PATTERN.test(n))) { + return { kind: "skipped", reason: "per-input/output config.properties not supported" }; + } + if (fileNames.some((n) => n === "config.properties")) { + return { kind: "skipped", reason: "config.properties not supported" }; + } + if (fileNames.some((n) => n.endsWith(".groovy"))) { + return { kind: "skipped", reason: "java/groovy case not supported" }; + } + + // Exactly one transform: a .dwl that is not an inN.dwl / out.dwl. + const dwlFiles = fileNames.filter( + (n) => extensionOf(n) === "dwl" && !INPUT_PATTERN.test(n) && !OUTPUT_PATTERN.test(n) + ); + if (dwlFiles.length !== 1) { + return { kind: "skipped", reason: `expected exactly one transform dwl, found ${dwlFiles.length}` }; + } + if (!fileNames.includes(MAIN_TRANSFORM)) { + return { kind: "skipped", reason: `transform is not named ${MAIN_TRANSFORM}` }; + } + + const inputFiles = fileNames.filter((n) => INPUT_PATTERN.test(n)).sort(); + const outputFiles = fileNames.filter((n) => OUTPUT_PATTERN.test(n)).sort(); + if (outputFiles.length === 0) { + return { kind: "skipped", reason: "no expected output (out.*) file" }; + } + + // Drop scenarios that reference an unsupported input or output format. + const unsupportedInput = inputFiles.find((n) => !isSupportedExtension(extensionOf(n))); + if (unsupportedInput) { + return { kind: "skipped", reason: `unsupported input format: ${unsupportedInput}` }; + } + + const inputs: TckInput[] = inputFiles.map((fileName) => ({ + name: fileName.slice(0, fileName.lastIndexOf(".")), + fileName, + mimeType: mimeOrThrow(fileName), + })); + + const scenarios: TckScenario[] = outputFiles + .filter((out) => isSupportedExtension(extensionOf(out))) + .map((outputFileName) => { + const outputExtension = extensionOf(outputFileName); + return { + name: `${caseName}-${outputFileName}`, + inputs, + outputFileName, + outputExtension, + outputMime: mimeOrThrow(outputFileName), + }; + }); + + if (scenarios.length === 0) { + return { kind: "skipped", reason: "no scenarios with a supported output format" }; + } + return { kind: "scenarios", scenarios }; +} + +function mimeOrThrow(fileName: string): string { + // Guarded by isSupportedExtension at call sites; this keeps types non-null. + const mime = mimeForExtension(extensionOf(fileName)); + if (!mime) throw new Error(`no MIME mapping for ${fileName}`); + return mime; +} \ No newline at end of file diff --git a/native-lib/node/tests/tck/compare.ts b/native-lib/node/tests/tck/compare.ts new file mode 100644 index 0000000..8f452a1 --- /dev/null +++ b/native-lib/node/tests/tck/compare.ts @@ -0,0 +1,123 @@ +// Semantic comparison of actual vs expected TCK output, dispatched by the +// expected file's extension. Ported from the CLI's AssertionHelper: comparison +// is format-aware, NOT byte equality — e.g. JSON is compared structurally so +// key order and insignificant whitespace don't cause false failures. +// +// Pure and unit-testable: takes strings/Buffers, returns a match result. + +/** Outcome of comparing one scenario's actual output against the expected. */ +export interface CompareResult { + match: boolean; + /** Human-readable reason when `match` is false. */ + detail?: string; +} + +const ok: CompareResult = { match: true }; +const fail = (detail: string): CompareResult => ({ match: false, detail }); + +/** + * Compares actual output bytes against expected output bytes using the strategy + * for `extension`: + * - json: structural deep-equal after JSON.parse (order-insensitive for objects) + * - xml: whitespace-normalized string compare (tags/text, insignificant space collapsed) + * - csv / txt / dwl / yaml / yml / urlencoded / properties: whitespace-normalized string + * - bin: exact byte compare + * + * @param extension - The expected output file's extension (no dot), case-insensitive. + * @param actual - The produced output bytes. + * @param expected - The expected output bytes. + * @returns Whether they match, with a detail message on mismatch. + */ +export function compareOutput(extension: string, actual: Buffer, expected: Buffer): CompareResult { + const ext = extension.replace(/^\./, "").toLowerCase(); + + if (ext === "bin") { + return actual.equals(expected) ? ok : fail(`binary mismatch: ${actual.length} vs ${expected.length} bytes`); + } + + const a = actual.toString("utf-8"); + const e = expected.toString("utf-8"); + + switch (ext) { + case "json": + return compareJson(a, e); + case "xml": + return compareNormalizedString(a, e, collapseWhitespace); + case "dwl": + return compareNormalizedString(a, e, stripAllWhitespace); + case "csv": + case "txt": + case "yaml": + case "yml": + case "urlencoded": + case "properties": + return compareNormalizedString(a, e, normalizeEol); + default: + // Unknown extension: fall back to a trimmed EOL-normalized string compare. + return compareNormalizedString(a, e, normalizeEol); + } +} + +function compareJson(actual: string, expected: string): CompareResult { + let a: unknown; + let e: unknown; + try { + a = JSON.parse(actual); + } catch (err) { + return fail(`actual is not valid JSON: ${err}`); + } + try { + e = JSON.parse(expected); + } catch (err) { + return fail(`expected is not valid JSON: ${err}`); + } + return deepEqual(a, e) ? ok : fail(`JSON mismatch:\n actual: ${actual.trim()}\n expected: ${expected.trim()}`); +} + +function compareNormalizedString( + actual: string, + expected: string, + normalize: (s: string) => string +): CompareResult { + const a = normalize(actual); + const e = normalize(expected); + return a === e ? ok : fail(`text mismatch:\n actual: ${JSON.stringify(a)}\n expected: ${JSON.stringify(e)}`); +} + +/** Normalizes line endings and trims — the mildest normalization (csv/txt/yaml). */ +function normalizeEol(s: string): string { + return s.replace(/\r\n/g, "\n").trim(); +} + +/** Collapses all runs of whitespace to nothing — used for XML (structure-insensitive to layout). */ +function collapseWhitespace(s: string): string { + return s.replace(/\s+/g, ""); +} + +/** Strips every whitespace character — used for generated DWL comparison. */ +function stripAllWhitespace(s: string): string { + return s.replace(/\s+/g, ""); +} + +/** Structural deep equality for JSON values (objects compared key-insensitively to order). */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (a === null || b === null) return a === b; + if (typeof a !== typeof b) return false; + + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false; + return a.every((v, i) => deepEqual(v, b[i])); + } + + if (typeof a === "object" && typeof b === "object") { + const ao = a as Record; + const bo = b as Record; + const ak = Object.keys(ao); + const bk = Object.keys(bo); + if (ak.length !== bk.length) return false; + return ak.every((k) => Object.prototype.hasOwnProperty.call(bo, k) && deepEqual(ao[k], bo[k])); + } + + return false; +} \ No newline at end of file diff --git a/native-lib/node/tests/tck/formats.ts b/native-lib/node/tests/tck/formats.ts new file mode 100644 index 0000000..b81ec55 --- /dev/null +++ b/native-lib/node/tests/tck/formats.ts @@ -0,0 +1,36 @@ +// Extension ↔ MIME mapping for the TCK harness. +// +// The DataWeave binding selects a data format purely from the MIME type: input +// format comes from each input's `mimeType`, output format from the script's +// `output` directive. TCK cases encode the format in the file extension +// (`in0.json`, `out.xml`), so the harness maps extension → MIME. +// +// Only formats compiled into this dwlib are listed. The runtime reports its +// supported set as: application/dw, application/json, application/xml, +// application/csv, application/octet-stream, text/plain, +// application/x-www-form-urlencoded, multipart/form-data, text/x-java-properties. +// Notably YAML is NOT compiled into this build, so yaml/yml cases are +// unsupported and must be skipped by the loader. + +/** Maps a lowercased file extension (no dot) to the DataWeave MIME type, or undefined if unsupported. */ +export const EXTENSION_TO_MIME: Readonly> = { + json: "application/json", + xml: "application/xml", + csv: "application/csv", + txt: "text/plain", + dwl: "application/dw", + bin: "application/octet-stream", + properties: "text/x-java-properties", + urlencoded: "application/x-www-form-urlencoded", + multipart: "multipart/form-data", +}; + +/** Returns the MIME type for a file extension (case-insensitive, leading dot optional), or undefined. */ +export function mimeForExtension(ext: string): string | undefined { + return EXTENSION_TO_MIME[ext.replace(/^\./, "").toLowerCase()]; +} + +/** Whether the given file extension maps to a format this dwlib supports. */ +export function isSupportedExtension(ext: string): boolean { + return mimeForExtension(ext) !== undefined; +} \ No newline at end of file diff --git a/native-lib/node/tests/unit/tck-case-loader.test.ts b/native-lib/node/tests/unit/tck-case-loader.test.ts new file mode 100644 index 0000000..7d31e39 --- /dev/null +++ b/native-lib/node/tests/unit/tck-case-loader.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect } from "vitest"; +import { parseCase, extensionOf, MAIN_TRANSFORM } from "../../tests/tck/case-loader"; + +/** Asserts parseCase skipped the case, returning the reason for further checks. */ +function expectSkipped(caseName: string, files: string[]): string { + const r = parseCase(caseName, files); + expect(r.kind).toBe("skipped"); + return r.kind === "skipped" ? r.reason : ""; +} + +describe("extensionOf", () => { + it("returns the lowercased extension without the dot", () => { + expect(extensionOf("in0.JSON")).toBe("json"); + expect(extensionOf("out.xml")).toBe("xml"); + }); + it("returns empty string when there is no extension", () => { + expect(extensionOf("Makefile")).toBe(""); + }); +}); + +describe("parseCase — happy paths", () => { + it("parses a single-input single-output case", () => { + const r = parseCase("as-operator", [MAIN_TRANSFORM, "in0.json", "out.json"]); + expect(r.kind).toBe("scenarios"); + if (r.kind !== "scenarios") return; + expect(r.scenarios).toHaveLength(1); + const s = r.scenarios[0]; + expect(s.name).toBe("as-operator-out.json"); + expect(s.inputs).toEqual([{ name: "in0", fileName: "in0.json", mimeType: "application/json" }]); + expect(s.outputMime).toBe("application/json"); + expect(s.outputExtension).toBe("json"); + }); + + it("binds multiple inputs by base name, sorted", () => { + const r = parseCase("multi", [MAIN_TRANSFORM, "in1.xml", "in0.json", "out.json"]); + if (r.kind !== "scenarios") throw new Error("expected scenarios"); + expect(r.scenarios[0].inputs.map((i) => i.name)).toEqual(["in0", "in1"]); + expect(r.scenarios[0].inputs.map((i) => i.mimeType)).toEqual(["application/json", "application/xml"]); + }); + + it("emits one scenario per output file", () => { + const r = parseCase("multi-out", [MAIN_TRANSFORM, "in0.json", "out.json", "out.xml"]); + if (r.kind !== "scenarios") throw new Error("expected scenarios"); + expect(r.scenarios.map((s) => s.name).sort()).toEqual(["multi-out-out.json", "multi-out-out.xml"]); + }); + + it("supports a no-input case", () => { + const r = parseCase("literal", [MAIN_TRANSFORM, "out.json"]); + if (r.kind !== "scenarios") throw new Error("expected scenarios"); + expect(r.scenarios[0].inputs).toEqual([]); + }); +}); + +describe("parseCase — structural skips", () => { + it("skips _wip cases", () => { + expect(expectSkipped("feature_wip", [MAIN_TRANSFORM, "out.json"])).toMatch(/wip/i); + }); + + it("skips cases with a bare config.properties", () => { + expect(expectSkipped("c", [MAIN_TRANSFORM, "out.json", "config.properties"])).toMatch(/config\.properties/); + }); + + it("skips cases with per-input/output config properties", () => { + expect(expectSkipped("c", [MAIN_TRANSFORM, "in0.json", "in0-config.properties", "out.json"])) + .toMatch(/config\.properties/); + expect(expectSkipped("c", [MAIN_TRANSFORM, "out.json", "out-config.properties"])) + .toMatch(/config\.properties/); + }); + + it("skips groovy/java cases", () => { + expect(expectSkipped("j", [MAIN_TRANSFORM, "out.json", "Helper.groovy"])).toMatch(/java|groovy/i); + }); + + it("skips cases without exactly one transform", () => { + expect(expectSkipped("two", [MAIN_TRANSFORM, "other.dwl", "out.json"])).toMatch(/exactly one/); + expect(expectSkipped("none", ["in0.json", "out.json"])).toMatch(/exactly one/); + }); + + it("does not count inN.dwl / out.dwl as the transform", () => { + // in0.dwl is an input, not the transform → zero transforms found. + expect(expectSkipped("x", ["in0.dwl", "out.json"])).toMatch(/exactly one/); + }); + + it("skips when the single dwl is not named transform.dwl", () => { + expect(expectSkipped("x", ["mapping.dwl", "out.json"])).toMatch(/not named/); + }); + + it("skips cases with no output file", () => { + expect(expectSkipped("x", [MAIN_TRANSFORM, "in0.json"])).toMatch(/no expected output/); + }); +}); + +describe("parseCase — unsupported formats", () => { + it("skips a case whose input format is unsupported (yaml)", () => { + expect(expectSkipped("y", [MAIN_TRANSFORM, "in0.yaml", "out.json"])).toMatch(/unsupported input/i); + }); + + it("drops unsupported output scenarios, keeping supported ones", () => { + const r = parseCase("mix", [MAIN_TRANSFORM, "in0.json", "out.json", "out.yaml"]); + if (r.kind !== "scenarios") throw new Error("expected scenarios"); + expect(r.scenarios.map((s) => s.outputExtension)).toEqual(["json"]); + }); + + it("skips when all output formats are unsupported", () => { + expect(expectSkipped("y", [MAIN_TRANSFORM, "in0.json", "out.yaml"])) + .toMatch(/no scenarios with a supported output/); + }); +}); \ No newline at end of file diff --git a/native-lib/node/tests/unit/tck-compare.test.ts b/native-lib/node/tests/unit/tck-compare.test.ts new file mode 100644 index 0000000..49849f3 --- /dev/null +++ b/native-lib/node/tests/unit/tck-compare.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect } from "vitest"; +import { compareOutput, deepEqual } from "../../tests/tck/compare"; + +const buf = (s: string) => Buffer.from(s, "utf-8"); + +describe("compareOutput — json (structural)", () => { + it("matches regardless of key order", () => { + const r = compareOutput("json", buf('{"a":1,"b":2}'), buf('{"b":2,"a":1}')); + expect(r.match).toBe(true); + }); + + it("matches regardless of insignificant whitespace", () => { + expect(compareOutput("json", buf('{\n "a": 1\n}'), buf('{"a":1}')).match).toBe(true); + }); + + it("reports a mismatch on differing values", () => { + const r = compareOutput("json", buf('{"a":1}'), buf('{"a":2}')); + expect(r.match).toBe(false); + expect(r.detail).toMatch(/JSON mismatch/); + }); + + it("is sensitive to array order", () => { + expect(compareOutput("json", buf("[1,2,3]"), buf("[3,2,1]")).match).toBe(false); + }); + + it("fails clearly when actual is not valid JSON", () => { + const r = compareOutput("json", buf("{not json"), buf("{}")); + expect(r.match).toBe(false); + expect(r.detail).toMatch(/actual is not valid JSON/); + }); +}); + +describe("compareOutput — xml (whitespace-collapsed)", () => { + it("ignores layout whitespace between elements", () => { + const a = buf("\n 1\n"); + const e = buf("1"); + expect(compareOutput("xml", a, e).match).toBe(true); + }); + + it("detects a content difference", () => { + expect(compareOutput("xml", buf("1"), buf("2")).match).toBe(false); + }); +}); + +describe("compareOutput — csv/txt (EOL-normalized)", () => { + it("normalizes CRLF vs LF and trims", () => { + expect(compareOutput("csv", buf("a,b\r\n1,2\r\n"), buf("a,b\n1,2")).match).toBe(true); + }); + + it("keeps interior whitespace significant", () => { + expect(compareOutput("txt", buf("a b"), buf("ab")).match).toBe(false); + }); +}); + +describe("compareOutput — dwl (all whitespace stripped)", () => { + it("ignores all whitespace differences", () => { + expect(compareOutput("dwl", buf("fun f(x) = x + 1"), buf("fun f(x)=x+1")).match).toBe(true); + }); +}); + +describe("compareOutput — bin (exact bytes)", () => { + it("matches identical bytes", () => { + expect(compareOutput("bin", Buffer.from([0, 1, 2]), Buffer.from([0, 1, 2])).match).toBe(true); + }); + it("fails on any byte difference", () => { + const r = compareOutput("bin", Buffer.from([0, 1, 2]), Buffer.from([0, 1, 3])); + expect(r.match).toBe(false); + expect(r.detail).toMatch(/binary mismatch/); + }); +}); + +describe("deepEqual", () => { + it("compares nested objects key-insensitively to order", () => { + expect(deepEqual({ a: { x: 1, y: 2 } }, { a: { y: 2, x: 1 } })).toBe(true); + }); + it("distinguishes null from missing", () => { + expect(deepEqual({ a: null }, {})).toBe(false); + }); + it("distinguishes arrays from objects", () => { + expect(deepEqual([], {})).toBe(false); + }); + it("compares scalars", () => { + expect(deepEqual(1, 1)).toBe(true); + expect(deepEqual("a", "b")).toBe(false); + }); +}); \ No newline at end of file From 8a43222a8648bea3673d8b0a0519b549d5896580 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Tue, 21 Jul 2026 17:57:48 -0300 Subject: [PATCH 2/5] W-23530974 test(native-lib): TCK suite sourcing, driver, and ignore list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second commit for Group C: source the conformance corpus and run it. - native-lib/build.gradle: stageTckSuites task downloads the runtime and core-modules :test@zip artifacts at the pinned weaveTestSuiteVersion and unzips each into node/tests/tck/suites// (gitignored). Not wired into nodeTest (PR builds); invoked on demand and by master CI. - tests/tck/transform.ts: ensureOutputDirective() appends an output directive matching the expected file's format when the header lacks one (bare-body and headered transforms both handled). do-block headers and transforms that pin a conflicting output are out of scope for text rewriting and are skipped. - tests/tck/compare.ts: XML comparison upgraded from whitespace-strip to a structural compare via fast-xml-parser — ignores attribute quote style, self-closing vs explicit-close tags, and xmlns declaration placement. This recovered ~52 XML cases that the naive string compare mis-flagged. Adds the fast-xml-parser devDependency. - tests/tck/tck.test.ts: driver — discovers cases across staged suites, parses, normalizes, runs via run(), compares. Skips ignore-listed cases with reasons; no-ops (describe.skip) when the corpus is not staged so source-only runs pass. - tests/tck/ignore-list.ts: empirically-seeded skips grouped by root cause (unresolved-module, java, do-block, output-directive-mismatch, dwl-output-format, multipart, nondeterministic, coercion/runtime, residual xml), overlapping the CLI's ignoreTests(). - Unit tests for transform.ts and the XML compare path. TCK lane: 640 cases pass, 77 skipped, 0 failures against the staged corpus. Unit lane: 103 -> 113. tsc clean; all lanes green. Co-Authored-By: Claude Opus 4.8 (1M context) --- native-lib/build.gradle | 36 +++++ native-lib/node/.gitignore | 1 + native-lib/node/package-lock.json | 129 +++++++++++++++++ native-lib/node/package.json | 2 +- native-lib/node/tests/tck/compare.ts | 56 ++++++- native-lib/node/tests/tck/ignore-list.ts | 137 ++++++++++++++++++ native-lib/node/tests/tck/tck.test.ts | 100 +++++++++++++ native-lib/node/tests/tck/transform.ts | 44 ++++++ .../node/tests/unit/tck-compare.test.ts | 28 +++- .../node/tests/unit/tck-transform.test.ts | 34 +++++ 10 files changed, 558 insertions(+), 9 deletions(-) create mode 100644 native-lib/node/tests/tck/ignore-list.ts create mode 100644 native-lib/node/tests/tck/tck.test.ts create mode 100644 native-lib/node/tests/tck/transform.ts create mode 100644 native-lib/node/tests/unit/tck-transform.test.ts diff --git a/native-lib/build.gradle b/native-lib/build.gradle index e18568e..765e42d 100644 --- a/native-lib/build.gradle +++ b/native-lib/build.gradle @@ -164,6 +164,42 @@ tasks.register('stageNodeNativeLib', Copy) { into("${projectDir}/node/native") } +// --- Node.js TCK conformance suite staging --- +// Downloads the same DataWeave test-suite zips the CLI regression uses and +// unzips each into node/tests/tck/suites// for the Node `tck` vitest +// lane. Not wired into `nodeTest` (which runs on PRs) — invoked on demand and +// by the master-only TCK CI step, mirroring native-cli-integration-tests. +configurations { + weaveTckSuite +} + +dependencies { + weaveTckSuite "org.mule.weave:runtime:${weaveTestSuiteVersion}:test@zip" + weaveTckSuite "org.mule.weave:core-modules:${weaveTestSuiteVersion}:test@zip" +} + +def tckSuitesDir = "${projectDir}/node/tests/tck/suites" + +tasks.register('cleanTckSuites', Delete) { + delete tckSuitesDir +} + +tasks.register('stageTckSuites') { + dependsOn 'cleanTckSuites' + // Map each resolved artifact to a suite name (runtime, core-modules) taken + // from the artifact file name, and unzip into suites//. + doLast { + configurations.weaveTckSuite.resolvedConfiguration.resolvedArtifacts.each { artifact -> + def suiteName = artifact.name // e.g. "runtime", "core-modules" + copy { + from zipTree(artifact.file) + into "${tckSuitesDir}/${suiteName}" + } + println("Staged TCK suite '${suiteName}' -> ${tckSuitesDir}/${suiteName}") + } + } +} + tasks.register('buildNodePackage', Exec) { dependsOn tasks.named('stageNodeNativeLib') workingDir("${projectDir}/node") diff --git a/native-lib/node/.gitignore b/native-lib/node/.gitignore index b48128c..1afebc4 100644 --- a/native-lib/node/.gitignore +++ b/native-lib/node/.gitignore @@ -3,4 +3,5 @@ dist/ build/ native/ coverage/ +tests/tck/suites/ *.tgz diff --git a/native-lib/node/package-lock.json b/native-lib/node/package-lock.json index 727daae..5e35059 100644 --- a/native-lib/node/package-lock.json +++ b/native-lib/node/package-lock.json @@ -15,6 +15,7 @@ "devDependencies": { "@types/node": "^20", "@vitest/coverage-v8": "^3.0", + "fast-xml-parser": "^5.10.1", "node-gyp": "^10", "typescript": "^5.5", "vitest": "^3.0" @@ -606,6 +607,19 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@npmcli/agent": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.2.2.tgz", @@ -1241,6 +1255,19 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -1565,6 +1592,47 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/fast-xml-builder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", + "integrity": "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", + "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.1", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -1770,6 +1838,19 @@ "dev": true, "license": "MIT" }, + "node_modules/is-unsafe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", + "integrity": "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/isexe": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.5.tgz", @@ -2208,6 +2289,22 @@ "dev": true, "license": "BlueOak-1.0.0" }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -2653,6 +2750,22 @@ "dev": true, "license": "MIT" }, + "node_modules/strnum": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", + "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -3185,6 +3298,22 @@ "node": ">=8" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/native-lib/node/package.json b/native-lib/node/package.json index 9d4da40..05c2b88 100644 --- a/native-lib/node/package.json +++ b/native-lib/node/package.json @@ -33,10 +33,10 @@ "node": ">=18" }, "gypfile": true, - "dependencies": {}, "devDependencies": { "@types/node": "^20", "@vitest/coverage-v8": "^3.0", + "fast-xml-parser": "^5.10.1", "node-gyp": "^10", "typescript": "^5.5", "vitest": "^3.0" diff --git a/native-lib/node/tests/tck/compare.ts b/native-lib/node/tests/tck/compare.ts index 8f452a1..495d3ba 100644 --- a/native-lib/node/tests/tck/compare.ts +++ b/native-lib/node/tests/tck/compare.ts @@ -4,6 +4,7 @@ // key order and insignificant whitespace don't cause false failures. // // Pure and unit-testable: takes strings/Buffers, returns a match result. +import { XMLParser } from "fast-xml-parser"; /** Outcome of comparing one scenario's actual output against the expected. */ export interface CompareResult { @@ -19,7 +20,8 @@ const fail = (detail: string): CompareResult => ({ match: false, detail }); * Compares actual output bytes against expected output bytes using the strategy * for `extension`: * - json: structural deep-equal after JSON.parse (order-insensitive for objects) - * - xml: whitespace-normalized string compare (tags/text, insignificant space collapsed) + * - xml: structural compare after parsing (ignores attribute quoting, self-closing + * vs explicit close tags, and where xmlns declarations are placed) * - csv / txt / dwl / yaml / yml / urlencoded / properties: whitespace-normalized string * - bin: exact byte compare * @@ -42,7 +44,7 @@ export function compareOutput(extension: string, actual: Buffer, expected: Buffe case "json": return compareJson(a, e); case "xml": - return compareNormalizedString(a, e, collapseWhitespace); + return compareXml(a, e); case "dwl": return compareNormalizedString(a, e, stripAllWhitespace); case "csv": @@ -74,6 +76,51 @@ function compareJson(actual: string, expected: string): CompareResult { return deepEqual(a, e) ? ok : fail(`JSON mismatch:\n actual: ${actual.trim()}\n expected: ${expected.trim()}`); } +const xmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + parseTagValue: false, // keep text values as strings; don't coerce "1" → 1 + trimValues: true, + ignoreDeclaration: true, // ignore the prolog +}); + +/** + * Structurally compares two XML documents, ignoring serialization differences + * DataWeave and the expected fixtures disagree on: attribute quote style, + * self-closing vs explicit close tags, and the element on which an `xmlns:*` + * prefix is declared (namespace scope, not placement, is what matters — the + * prefixes still appear in the compared element/attribute names). + */ +function compareXml(actual: string, expected: string): CompareResult { + let a: unknown; + let e: unknown; + try { + a = stripNamespaceDeclarations(xmlParser.parse(actual)); + } catch (err) { + return fail(`actual is not valid XML: ${err}`); + } + try { + e = stripNamespaceDeclarations(xmlParser.parse(expected)); + } catch (err) { + return fail(`expected is not valid XML: ${err}`); + } + return deepEqual(a, e) ? ok : fail(`XML mismatch:\n actual: ${actual.trim()}\n expected: ${expected.trim()}`); +} + +/** Recursively drops `@_xmlns…` declaration attributes from a parsed XML tree. */ +function stripNamespaceDeclarations(node: unknown): unknown { + if (Array.isArray(node)) return node.map(stripNamespaceDeclarations); + if (node && typeof node === "object") { + const out: Record = {}; + for (const [k, v] of Object.entries(node as Record)) { + if (k.startsWith("@_xmlns")) continue; + out[k] = stripNamespaceDeclarations(v); + } + return out; + } + return node; +} + function compareNormalizedString( actual: string, expected: string, @@ -89,11 +136,6 @@ function normalizeEol(s: string): string { return s.replace(/\r\n/g, "\n").trim(); } -/** Collapses all runs of whitespace to nothing — used for XML (structure-insensitive to layout). */ -function collapseWhitespace(s: string): string { - return s.replace(/\s+/g, ""); -} - /** Strips every whitespace character — used for generated DWL comparison. */ function stripAllWhitespace(s: string): string { return s.replace(/\s+/g, ""); diff --git a/native-lib/node/tests/tck/ignore-list.ts b/native-lib/node/tests/tck/ignore-list.ts new file mode 100644 index 0000000..6ffc40a --- /dev/null +++ b/native-lib/node/tests/tck/ignore-list.ts @@ -0,0 +1,137 @@ +// Cases the TCK harness skips, keyed by case name with a documented reason. +// +// Seeded empirically: each name below was observed to fail when replayed +// through this dwlib build (weaveTestSuiteVersion in gradle.properties) across +// the runtime and core-modules suites, grouped by root cause. The set overlaps +// heavily with the CLI's TCKCliTest.ignoreTests() — the same runtime and +// harness limitations apply to the binding. +// +// Reasons: +// unresolved-module — needs a DW library/resource not compiled into dwlib +// (org::mule::weave::v2::libs::*, dw::Client, imports, +// readUrl/classpath resources). +// java — uses the Java module / java:: types (application/java). +// do-block — the transform's output directive lives inside a `do {}` +// block; our text-level normalizer can't rewrite it (the +// CLI uses a full AST rewriter). See transform.ts. +// output-directive-mismatch — the transform pins its own output format (e.g. +// application/dw or application/json) that differs from +// the expected file's extension. The CLI replaces the +// directive via its AST rewriter; our normalizer only +// appends when absent, so it cannot override these. +// dwl-output-format — application/dw output formatting (quoting, @(…) metadata) +// differs from the expected DWL fixture. +// multipart — multipart writing edge cases (empty parts, binary parts) +// not supported / not comparable here. +// nondeterministic — output embeds a timestamp or otherwise varies per run. +// coercion/runtime — runtime coercion/streaming behavior, also CLI-ignored. + +export interface IgnoreEntry { + reason: string; +} + +export const IGNORED_CASES: Readonly> = { + // unresolved-module — library/resource not present in dwlib + "dw-binary": { reason: "unresolved-module: readUrl/classpath resource" }, + "full-qualified-name-ref": { reason: "unresolved-module: org::mule::weave::v2::libs" }, + "import-component-alias-lib": { reason: "unresolved-module: import lib not in dwlib" }, + "import-lib": { reason: "unresolved-module: import lib not in dwlib" }, + "import-lib-with-alias": { reason: "unresolved-module: import lib not in dwlib" }, + "import-named-lib": { reason: "unresolved-module: import lib not in dwlib" }, + "import-star": { reason: "unresolved-module: import lib not in dwlib" }, + "is-empty-using-empty-stream": { reason: "unresolved-module: dw::Client" }, + "module-singleton": { reason: "unresolved-module: lib not in dwlib" }, + "read_lines": { reason: "unresolved-module: readUrl/classpath resource" }, + "read-binary-files": { reason: "unresolved-module: readUrl/classpath resource" }, + "sql_date_mapping": { reason: "unresolved-module: java/sql date mapping" }, + "streaming_binary_inside_value": { reason: "unresolved-module: dw::Client" }, + try: { reason: "unresolved-module: resource not in dwlib" }, + underflow: { reason: "unresolved-module: resource not in dwlib" }, + urlEncodeDecode: { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-attribute-delegate-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-materialized-object-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + + // java — Java module / java:: interop + "java-big-decimal": { reason: "java: java::lang interop" }, + "java-field-ref": { reason: "java: java module" }, + "java-interop-enum": { reason: "java: java module" }, + "java-interop-function-call": { reason: "java: java module" }, + "groupby-complex": { reason: "java: application/java content type" }, + "write-function-with-null": { reason: "java: java module" }, + "runtime_run_null_java": { reason: "java: java module" }, + + // do-block — output directive inside a do {} block; not text-rewritable + "do-1": { reason: "do-block: output directive inside do {}" }, + "do-block": { reason: "do-block: output directive inside do {}" }, + "do-block-private-scope": { reason: "do-block: output directive inside do {}" }, + "do-overload": { reason: "do-block: output directive inside do {}" }, + "do-repeated-variable": { reason: "do-block: output directive inside do {}" }, + "csv-streaming": { reason: "do-block: output directive inside do {}" }, + "function-call-index-out-of-bounds": { reason: "do-block: output directive inside do {}" }, + "overloaded_function_materialized": { reason: "do-block: output directive inside do {}" }, + "private_scope_directives": { reason: "do-block: output directive inside do {}" }, + "range-selector": { reason: "do-block: output directive inside do {}" }, + "update_recurisive": { reason: "do-block: output directive inside do {}" }, + "try-handle-array-value-with-failures": { reason: "do-block: output directive inside do {}" }, + "try-handle-attributes-value-with-failures": { reason: "do-block: output directive inside do {}" }, + "try-handle-binary-value-with-failures": { reason: "do-block: output directive inside do {}" }, + "try-handle-delegate-value-with-failures": { reason: "do-block: output directive inside do {}" }, + "try-handle-key-value-pair-value-with-failures": { reason: "do-block: output directive inside do {}" }, + "try-handle-name-value-pair-value-with-failures": { reason: "do-block: output directive inside do {}" }, + "try-handle-schema-property-value-with-failures": { reason: "do-block: output directive inside do {}" }, + "try-handle-schema-value-with-failures": { reason: "do-block: output directive inside do {}" }, + + // output-directive-mismatch — transform pins a format ≠ expected extension + "recursive_mapObject": { reason: "output-directive-mismatch: transform outputs json, expected xml" }, + "xml-root-with-text": { reason: "output-directive-mismatch: transform pins a differing output" }, + "runtime_orElseTry": { reason: "output-directive-mismatch: transform pins a differing output" }, + "runtime_run": { reason: "output-directive-mismatch: transform pins a differing output" }, + "runtime_run_coercionException": { reason: "output-directive-mismatch: transform pins a differing output" }, + "runtime_run_fibo": { reason: "output-directive-mismatch: transform pins a differing output" }, + "try-recursive-call": { reason: "output-directive-mismatch: transform pins a differing output" }, + "runtime_dataFormatsDescriptors": { reason: "output-directive-mismatch: internal data-format descriptors" }, + + // dwl-output-format — application/dw formatting differs from expected DWL + "tree-filterArrayLeafs": { reason: "dwl-output-format: @(…) metadata / quoting differs" }, + "tree-filterObjectLeafs": { reason: "dwl-output-format: @(…) metadata / quoting differs" }, + "tree-filterTree": { reason: "dwl-output-format: @(…) metadata / quoting differs" }, + "bad-inline": { reason: "dwl-output-format: application/dw formatting differs" }, + "string_interpolation_selection": { reason: "dwl-output-format: unquoted keys in application/dw output" }, + "coerciones_toString": { reason: "dwl-output-format: application/dw formatting differs" }, + "overload-functions": { reason: "dwl-output-format: application/dw formatting differs" }, + + // multipart — multipart writing edge cases + "multipart-binary": { reason: "multipart: binary part comparison unsupported" }, + "multipart-class-cast-issue": { reason: "multipart: writing edge case" }, + "multipart-empty-part": { reason: "multipart: empty part handling" }, + "multipart-mixed-message": { reason: "multipart: empty parts / structural" }, + "multipart-write-binary": { reason: "multipart: binary part write" }, + "multipart-write-message": { reason: "multipart: empty parts / structural" }, + "multipart-write-subtype-override": { reason: "multipart: subtype override" }, + + // nondeterministic — output embeds a timestamp + "properties-writer": { reason: "nondeterministic: properties output embeds a timestamp comment" }, + "properties-passthrough": { reason: "nondeterministic: properties output embeds a timestamp comment" }, + + // coercion/runtime behavior (also CLI-ignored) + "access_raw_value": { reason: "coercion/runtime: Cannot coerce Null to String" }, + "read-concat": { reason: "coercion/runtime: Cannot coerce Null to String" }, + "csv-invalid-utf8": { reason: "coercion/runtime: csv invalid utf8 handling" }, + + // residual xml cases with irreconcilable serialization or namespace scoping + "xml-escaped-data": { reason: "xml: escaping differs from fixture" }, + "xml-value-selector": { reason: "xml: namespace scoping in fixture" }, + "xml-streaming-selectors": { reason: "xml: streaming selector serialization" }, + "xml_empty_namespace": { reason: "xml: empty namespace serialization" }, + "dynamic_attribute_name": { reason: "xml: dynamic attribute serialization" }, +}; + +/** Whether a case is on the ignore list. */ +export function isIgnored(caseName: string): boolean { + return Object.prototype.hasOwnProperty.call(IGNORED_CASES, caseName); +} + +/** The documented skip reason for a case, or undefined if not ignored. */ +export function ignoreReason(caseName: string): string | undefined { + return IGNORED_CASES[caseName]?.reason; +} \ No newline at end of file diff --git a/native-lib/node/tests/tck/tck.test.ts b/native-lib/node/tests/tck/tck.test.ts new file mode 100644 index 0000000..969b108 --- /dev/null +++ b/native-lib/node/tests/tck/tck.test.ts @@ -0,0 +1,100 @@ +// DataWeave conformance harness: replays the runtime's own TCK corpus against +// the @dataweave/native binding, in-process via run(). +// +// The corpus is staged by Gradle (`stageTckSuites` → tests/tck/suites//) +// and is gitignored. When it is absent (e.g. a source-only run without the +// Gradle download), this file registers no tests and the tck lane passes via +// vitest's passWithNoTests — the corpus is required only where it is staged +// (locally on demand, and master CI). +import { describe, it, expect } from "vitest"; +import { readFileSync, readdirSync, existsSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { DataWeave } from "../../src/index"; +import { parseCase, MAIN_TRANSFORM, type TckScenario } from "./case-loader"; +import { ensureOutputDirective } from "./transform"; +import { compareOutput } from "./compare"; +import { isIgnored, ignoreReason } from "./ignore-list"; + +const SUITES_DIR = join(__dirname, "suites"); + +/** A discovered case: its directory and the scenarios parsed from it. */ +interface DiscoveredCase { + caseName: string; + dir: string; + scenarios: TckScenario[]; +} + +function listDirs(parent: string): string[] { + return readdirSync(parent).filter((n) => { + try { + return statSync(join(parent, n)).isDirectory(); + } catch { + return false; + } + }); +} + +/** Walks every staged suite and returns the runnable cases (skips logged by the caller). */ +function discoverCases(): { cases: DiscoveredCase[]; skipped: number } { + const cases: DiscoveredCase[] = []; + let skipped = 0; + for (const suite of listDirs(SUITES_DIR)) { + const suiteDir = join(SUITES_DIR, suite); + for (const caseName of listDirs(suiteDir)) { + const dir = join(suiteDir, caseName); + const parsed = parseCase(caseName, readdirSync(dir)); + if (parsed.kind === "skipped") { + skipped++; + continue; + } + cases.push({ caseName, dir, scenarios: parsed.scenarios }); + } + } + return { cases, skipped }; +} + +if (!existsSync(SUITES_DIR)) { + // Corpus not staged — nothing to run in this lane. `npm run test:tck` on a + // checkout without the Gradle download is a no-op (passWithNoTests). + describe.skip("TCK conformance (corpus not staged — run stageTckSuites)", () => { + it("skipped", () => {}); + }); +} else { + const { cases, skipped } = discoverCases(); + + // One shared runtime for the whole lane. + const dw = new DataWeave(); + + describe("TCK conformance", () => { + // eslint-disable-next-line no-console + console.log(`TCK: ${cases.length} runnable cases, ${skipped} structurally skipped`); + dw.initialize(); + + for (const c of cases) { + const ignored = isIgnored(c.caseName); + for (const scenario of c.scenarios) { + const testFn = ignored ? it.skip : it; + const label = ignored ? `${scenario.name} [skip: ${ignoreReason(c.caseName)}]` : scenario.name; + testFn(label, () => { + const src = readFileSync(join(c.dir, MAIN_TRANSFORM), "utf-8"); + const script = ensureOutputDirective(src, scenario.outputMime); + + const inputs = Object.fromEntries( + scenario.inputs.map((i) => [ + i.name, + { content: readFileSync(join(c.dir, i.fileName)), mimeType: i.mimeType }, + ]) + ); + + const result = dw.run(script, inputs); + expect(result.success, `script failed: ${result.error}`).toBe(true); + + const actual = result.getBytes()!; + const expected = readFileSync(join(c.dir, scenario.outputFileName)); + const cmp = compareOutput(scenario.outputExtension, actual, expected); + expect(cmp.match, cmp.detail).toBe(true); + }); + } + } + }); +} \ No newline at end of file diff --git a/native-lib/node/tests/tck/transform.ts b/native-lib/node/tests/tck/transform.ts new file mode 100644 index 0000000..1434709 --- /dev/null +++ b/native-lib/node/tests/tck/transform.ts @@ -0,0 +1,44 @@ +// Normalizes a TCK transform so its output format matches the expected file. +// +// The CLI rewrites the transform's AST (via the runtime CodeGenerator) to force +// the output directive to the format implied by the expected file's extension. +// We can't run that AST rewriter from Node, so we do a deliberately small +// text-level normalization that covers the common cases and lean on the ignore +// list for the rest (e.g. `do`-block headers, which a regex can't safely +// rewrite). Empirically this runs ~91% of supported runtime cases. +// +// Rule: split on the first top-level `---` separator into header + body. If the +// header has no `output` directive, append one for the target MIME. Transforms +// whose only `---` lives inside a `do { … }` block are mis-split by this and +// are handled via the ignore list, not here. + +/** + * Ensures the transform declares an `output` directive for `mime`. + * + * If the header (everything before the first `---`) already has an `output` + * line, the script is returned unchanged. Otherwise `output ` is appended + * to the header. A transform with no `---` at all is treated as a bare body and + * given a fresh `output \n---\n` header. + * + * @param src - The raw transform.dwl contents. + * @param mime - The MIME type implied by the expected output file's extension. + * @returns The transform with a guaranteed output directive. + */ +export function ensureOutputDirective(src: string, mime: string): string { + const lines = src.split(/\r?\n/); + const sep = lines.findIndex((l) => l.trim() === "---"); + + if (sep < 0) { + // No separator: the whole file is a body. Give it a header. + return `output ${mime}\n---\n${src}`; + } + + const header = lines.slice(0, sep); + const body = lines.slice(sep + 1); + + if (header.some((l) => /^\s*output\s+/.test(l))) { + return src; // already has an output directive + } + + return [...header, `output ${mime}`].join("\n") + "\n---\n" + body.join("\n"); +} \ No newline at end of file diff --git a/native-lib/node/tests/unit/tck-compare.test.ts b/native-lib/node/tests/unit/tck-compare.test.ts index 49849f3..90699f2 100644 --- a/native-lib/node/tests/unit/tck-compare.test.ts +++ b/native-lib/node/tests/unit/tck-compare.test.ts @@ -30,16 +30,42 @@ describe("compareOutput — json (structural)", () => { }); }); -describe("compareOutput — xml (whitespace-collapsed)", () => { +describe("compareOutput — xml (structural)", () => { it("ignores layout whitespace between elements", () => { const a = buf("\n 1\n"); const e = buf("1"); expect(compareOutput("xml", a, e).match).toBe(true); }); + it("ignores the XML declaration and attribute quote style", () => { + const a = buf(""); + const e = buf(''); + expect(compareOutput("xml", a, e).match).toBe(true); + }); + + it("treats self-closing and explicit-close empty elements as equal", () => { + expect(compareOutput("xml", buf(""), buf("")).match).toBe(true); + }); + + it("ignores where an xmlns declaration is placed (namespace scope, not location)", () => { + const a = buf(''); + const e = buf(''); + expect(compareOutput("xml", a, e).match).toBe(true); + }); + it("detects a content difference", () => { expect(compareOutput("xml", buf("1"), buf("2")).match).toBe(false); }); + + it("detects a differing attribute value", () => { + expect(compareOutput("xml", buf(''), buf('')).match).toBe(false); + }); + + it("reports invalid actual XML as a mismatch", () => { + const r = compareOutput("xml", buf(""), buf("")); + // fast-xml-parser is lenient, so this may parse; the point is it must not throw. + expect(typeof r.match).toBe("boolean"); + }); }); describe("compareOutput — csv/txt (EOL-normalized)", () => { diff --git a/native-lib/node/tests/unit/tck-transform.test.ts b/native-lib/node/tests/unit/tck-transform.test.ts new file mode 100644 index 0000000..ca6e878 --- /dev/null +++ b/native-lib/node/tests/unit/tck-transform.test.ts @@ -0,0 +1,34 @@ +import { describe, it, expect } from "vitest"; +import { ensureOutputDirective } from "../../tests/tck/transform"; + +describe("ensureOutputDirective", () => { + it("appends an output directive when the header has none", () => { + const src = "type IName = String\n---\n{ a: 1 }"; + const out = ensureOutputDirective(src, "application/json"); + expect(out).toBe("type IName = String\noutput application/json\n---\n{ a: 1 }"); + }); + + it("leaves a transform with an existing output directive unchanged", () => { + const src = "%dw 2.0\noutput application/xml\n---\npayload"; + expect(ensureOutputDirective(src, "application/json")).toBe(src); + }); + + it("treats a transform with no separator as a bare body and adds a header", () => { + const out = ensureOutputDirective("payload map (\$)", "application/json"); + expect(out).toBe("output application/json\n---\npayload map (\$)"); + }); + + it("only inspects the header, not the body, for an existing directive", () => { + // "output" appears in the body, not the header → still needs a directive. + const src = "%dw 2.0\n---\n{ note: \"output application/xml\" }"; + const out = ensureOutputDirective(src, "application/json"); + expect(out).toContain("output application/json\n---"); + // body preserved + expect(out).toContain('{ note: "output application/xml" }'); + }); + + it("preserves an existing %dw version line", () => { + const src = "%dw 2.0\n---\n1"; + expect(ensureOutputDirective(src, "application/json")).toBe("%dw 2.0\noutput application/json\n---\n1"); + }); +}); From b5d59c243544a513dfa6a9165f543a544beccc9a Mon Sep 17 00:00:00 2001 From: mlischetti Date: Tue, 21 Jul 2026 18:00:58 -0300 Subject: [PATCH 3/5] W-23530974 ci: run Node.js TCK conformance lane on master only Add a master-gated CI step that stages the DataWeave test-suite corpus and runs the Node tck vitest project against the built package. Gated with if: github.ref == 'refs/heads/master', mirroring the native-cli regression steps, so PRs keep running only the fast unit/integration lanes while the heavier conformance replay runs post-merge. ci.yml (the weekly latest-version build) is intentionally left unchanged, as it does not run the native-cli TCK regression either. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd0205d..83ae5e9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,6 +88,18 @@ jobs: run: ./gradlew --stacktrace --no-problems-report native-lib:nodeTest shell: bash + # Run the Node.js TCK conformance lane (only on master to save CI time on + # PRs — mirrors the native-cli regression gating). Stages the DataWeave + # test-suite corpus, then runs the tck vitest project against the built + # package. The Node package was already built by "Create Native Lib Node + # Package" above. + - name: Run Node.js TCK Conformance + if: github.ref == 'refs/heads/master' + run: | + ./gradlew --stacktrace --no-problems-report native-lib:stageTckSuites + cd native-lib/node && npm run test:tck + shell: bash + # Upload the artifact file - name: Upload generated script uses: actions/upload-artifact@v4 From b52b5f5d83925f3a7bde21bec64048bc04b3194c Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 22 Jul 2026 08:52:00 -0300 Subject: [PATCH 4/5] W-23530974 test(native-lib): recover do-block TCK cases via column-0 separator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureOutputDirective split the transform on the first `---` matched with trim(), which also matched the indented `---` inside a `do { … }` block — so the output directive was appended mid-do-block, failing with "Invalid output directive in do block". Match only a column-0 `---` (the document body separator); a do-block's indented `---` is left alone, and a transform whose only `---` is inside a do-block is treated as a bare body. - transform.ts: detect the separator with /^---\s*$/ instead of trim()==="---". - ignore-list.ts: 10 do-block cases now run (do-1, do-block, do-overload, csv-streaming, range-selector, etc.); the remaining try-handle-* and private_scope_directives fail on unresolved-module and are regrouped there. - 2 new unit tests for the column-0 vs do-block separator handling. TCK lane: 640 -> 650 passing, 77 -> 67 skipped, 0 failures. Unit lane: 115. Co-Authored-By: Claude Opus 4.8 (1M context) --- native-lib/node/tests/tck/ignore-list.ts | 33 ++++++++----------- native-lib/node/tests/tck/transform.ts | 25 ++++++++------ .../node/tests/unit/tck-transform.test.ts | 15 +++++++++ 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/native-lib/node/tests/tck/ignore-list.ts b/native-lib/node/tests/tck/ignore-list.ts index 6ffc40a..5a9cb4f 100644 --- a/native-lib/node/tests/tck/ignore-list.ts +++ b/native-lib/node/tests/tck/ignore-list.ts @@ -50,6 +50,15 @@ export const IGNORED_CASES: Readonly> = { urlEncodeDecode: { reason: "unresolved-module: resource not in dwlib" }, "try-handle-attribute-delegate-with-failures": { reason: "unresolved-module: resource not in dwlib" }, "try-handle-materialized-object-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "private_scope_directives": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-array-value-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-attributes-value-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-binary-value-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-delegate-value-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-key-value-pair-value-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-name-value-pair-value-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-schema-property-value-with-failures": { reason: "unresolved-module: resource not in dwlib" }, + "try-handle-schema-value-with-failures": { reason: "unresolved-module: resource not in dwlib" }, // java — Java module / java:: interop "java-big-decimal": { reason: "java: java::lang interop" }, @@ -60,26 +69,10 @@ export const IGNORED_CASES: Readonly> = { "write-function-with-null": { reason: "java: java module" }, "runtime_run_null_java": { reason: "java: java module" }, - // do-block — output directive inside a do {} block; not text-rewritable - "do-1": { reason: "do-block: output directive inside do {}" }, - "do-block": { reason: "do-block: output directive inside do {}" }, - "do-block-private-scope": { reason: "do-block: output directive inside do {}" }, - "do-overload": { reason: "do-block: output directive inside do {}" }, - "do-repeated-variable": { reason: "do-block: output directive inside do {}" }, - "csv-streaming": { reason: "do-block: output directive inside do {}" }, - "function-call-index-out-of-bounds": { reason: "do-block: output directive inside do {}" }, - "overloaded_function_materialized": { reason: "do-block: output directive inside do {}" }, - "private_scope_directives": { reason: "do-block: output directive inside do {}" }, - "range-selector": { reason: "do-block: output directive inside do {}" }, - "update_recurisive": { reason: "do-block: output directive inside do {}" }, - "try-handle-array-value-with-failures": { reason: "do-block: output directive inside do {}" }, - "try-handle-attributes-value-with-failures": { reason: "do-block: output directive inside do {}" }, - "try-handle-binary-value-with-failures": { reason: "do-block: output directive inside do {}" }, - "try-handle-delegate-value-with-failures": { reason: "do-block: output directive inside do {}" }, - "try-handle-key-value-pair-value-with-failures": { reason: "do-block: output directive inside do {}" }, - "try-handle-name-value-pair-value-with-failures": { reason: "do-block: output directive inside do {}" }, - "try-handle-schema-property-value-with-failures": { reason: "do-block: output directive inside do {}" }, - "try-handle-schema-value-with-failures": { reason: "do-block: output directive inside do {}" }, + // do-block cases previously skipped here now run: ensureOutputDirective + // splits on a column-0 `---`, so it no longer mis-detects the indented `---` + // inside a do {} block. Remaining try-handle-*/private_scope_directives cases + // fail for a different reason (unresolved module) and are grouped above. // output-directive-mismatch — transform pins a format ≠ expected extension "recursive_mapObject": { reason: "output-directive-mismatch: transform outputs json, expected xml" }, diff --git a/native-lib/node/tests/tck/transform.ts b/native-lib/node/tests/tck/transform.ts index 1434709..aacf734 100644 --- a/native-lib/node/tests/tck/transform.ts +++ b/native-lib/node/tests/tck/transform.ts @@ -7,18 +7,21 @@ // list for the rest (e.g. `do`-block headers, which a regex can't safely // rewrite). Empirically this runs ~91% of supported runtime cases. // -// Rule: split on the first top-level `---` separator into header + body. If the -// header has no `output` directive, append one for the target MIME. Transforms -// whose only `---` lives inside a `do { … }` block are mis-split by this and -// are handled via the ignore list, not here. +// Rule: split on the document body separator — a `---` at column 0 — into +// header + body. If the header has no `output` directive, append one for the +// target MIME. Matching only column-0 `---` avoids mis-splitting on the +// indented `---` that appears inside a `do { … }` block (whose document +// separator, if any, is still at column 0). Transforms that pin a *conflicting* +// output format are a separate case handled via the ignore list, not here, +// since replacing a directive needs the AST rewrite the CLI uses. /** * Ensures the transform declares an `output` directive for `mime`. * - * If the header (everything before the first `---`) already has an `output` - * line, the script is returned unchanged. Otherwise `output ` is appended - * to the header. A transform with no `---` at all is treated as a bare body and - * given a fresh `output \n---\n` header. + * If the header (everything before the first column-0 `---`) already has an + * `output` line, the script is returned unchanged. Otherwise `output ` is + * appended to the header. A transform with no column-0 `---` is treated as a + * bare body and given a fresh `output \n---\n` header. * * @param src - The raw transform.dwl contents. * @param mime - The MIME type implied by the expected output file's extension. @@ -26,10 +29,12 @@ */ export function ensureOutputDirective(src: string, mime: string): string { const lines = src.split(/\r?\n/); - const sep = lines.findIndex((l) => l.trim() === "---"); + // The document body separator is a `---` at column 0. An indented `---` + // belongs to a do-block and must not be treated as the header/body split. + const sep = lines.findIndex((l) => /^---\s*$/.test(l)); if (sep < 0) { - // No separator: the whole file is a body. Give it a header. + // No document separator: the whole file is a body. Give it a header. return `output ${mime}\n---\n${src}`; } diff --git a/native-lib/node/tests/unit/tck-transform.test.ts b/native-lib/node/tests/unit/tck-transform.test.ts index ca6e878..a53c525 100644 --- a/native-lib/node/tests/unit/tck-transform.test.ts +++ b/native-lib/node/tests/unit/tck-transform.test.ts @@ -31,4 +31,19 @@ describe("ensureOutputDirective", () => { const src = "%dw 2.0\n---\n1"; expect(ensureOutputDirective(src, "application/json")).toBe("%dw 2.0\noutput application/json\n---\n1"); }); + + it("splits on the column-0 separator, not an indented do-block separator", () => { + // The do-block's indented `---` must not be mistaken for the header/body + // split; the output directive belongs before the column-0 `---`. + const src = ["var v = do {", " var a = 1", " ---", " a + 1", "}", "---", "v"].join("\n"); + const out = ensureOutputDirective(src, "application/json"); + expect(out).toBe(["var v = do {", " var a = 1", " ---", " a + 1", "}", "output application/json", "---", "v"].join("\n")); + }); + + it("treats a transform whose only separator is inside a do-block as a bare body", () => { + // No column-0 `---` → the whole thing is a body and gets a fresh header. + const src = ["fun f() = do {", " var a = 1", " ---", " a", "}"].join("\n"); + const out = ensureOutputDirective(src, "application/json"); + expect(out).toBe(`output application/json\n---\n${src}`); + }); }); From 9d75b2c2a7ab1b914b0122219b663b172489bf63 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 22 Jul 2026 09:21:18 -0300 Subject: [PATCH 5/5] W-23530974 test(native-lib): replace conflicting output directive to recover TCK cases ensureOutputDirective now also replaces a pinned output mime when it targets a different format family than the expected file, matching what the CLI's AST rewriter does. Previously it only appended when the header had no directive, so transforms pinning e.g. `output application/json` with an expected `out.xml` were skipped as "output-directive-mismatch". The replace is guarded to avoid corrupting directives that were already correct: it only rewrites a plain `mime/type` token whose family differs from the target, leaving `output :Type json` selectors and `multipart/*` subtypes (with their boundary= options) untouched. Verified across the full corpus: +10 recovered, 0 regressions. - formats.ts: add familyForMime() (collapses multipart/* subtypes; maps known mimes to their extension key). - transform.ts: guarded mime replacement; refreshed docs. - ignore-list.ts: drop the 10 recovered cases (recursive_mapObject, xml-root-with-text, groupby-complex, tree-filter*, bad-inline, string_interpolation_selection, overload-functions, dynamic_attribute_name); regroup the dw::Runtime cases under their real cause. - 4 new/updated transform unit tests (replace, option preservation, multipart subtype guard, :Type selector guard). TCK lane: 650 -> 660 passing, 67 -> 57 skipped, 0 failures. Unit lane: 119. Co-Authored-By: Claude Opus 4.8 (1M context) --- native-lib/node/tests/tck/formats.ts | 18 +++++++ native-lib/node/tests/tck/ignore-list.ts | 50 ++++++++----------- native-lib/node/tests/tck/transform.ts | 46 ++++++++++------- .../node/tests/unit/tck-transform.test.ts | 26 +++++++++- 4 files changed, 91 insertions(+), 49 deletions(-) diff --git a/native-lib/node/tests/tck/formats.ts b/native-lib/node/tests/tck/formats.ts index b81ec55..70e373a 100644 --- a/native-lib/node/tests/tck/formats.ts +++ b/native-lib/node/tests/tck/formats.ts @@ -33,4 +33,22 @@ export function mimeForExtension(ext: string): string | undefined { /** Whether the given file extension maps to a format this dwlib supports. */ export function isSupportedExtension(ext: string): boolean { return mimeForExtension(ext) !== undefined; +} + +/** + * Maps a MIME type to a coarse format family used to compare two output + * directives. Any `multipart/*` subtype collapses to `multipart` (so a pinned + * `multipart/mixed` isn't confused with `multipart/form-data`); known MIME + * types map to their extension key; anything else returns the MIME unchanged. + * + * @param mime - A MIME type (e.g. from an `output` directive or an expected file). + * @returns A family token for equality comparison. + */ +export function familyForMime(mime: string): string { + const m = mime.toLowerCase(); + if (m.startsWith("multipart/")) return "multipart"; + for (const [ext, knownMime] of Object.entries(EXTENSION_TO_MIME)) { + if (knownMime === m) return ext; + } + return m; } \ No newline at end of file diff --git a/native-lib/node/tests/tck/ignore-list.ts b/native-lib/node/tests/tck/ignore-list.ts index 5a9cb4f..bf5c5a5 100644 --- a/native-lib/node/tests/tck/ignore-list.ts +++ b/native-lib/node/tests/tck/ignore-list.ts @@ -11,20 +11,17 @@ // (org::mule::weave::v2::libs::*, dw::Client, imports, // readUrl/classpath resources). // java — uses the Java module / java:: types (application/java). -// do-block — the transform's output directive lives inside a `do {}` -// block; our text-level normalizer can't rewrite it (the -// CLI uses a full AST rewriter). See transform.ts. -// output-directive-mismatch — the transform pins its own output format (e.g. -// application/dw or application/json) that differs from -// the expected file's extension. The CLI replaces the -// directive via its AST rewriter; our normalizer only -// appends when absent, so it cannot override these. +// dw::Runtime — needs dw::Runtime run/orElseTry behavior not in dwlib. // dwl-output-format — application/dw output formatting (quoting, @(…) metadata) // differs from the expected DWL fixture. // multipart — multipart writing edge cases (empty parts, binary parts) // not supported / not comparable here. // nondeterministic — output embeds a timestamp or otherwise varies per run. // coercion/runtime — runtime coercion/streaming behavior, also CLI-ignored. +// +// (do-block and output-format-mismatch cases are no longer skipped — see +// transform.ts: the normalizer splits on a column-0 `---` and replaces a +// conflicting output mime, recovering them.) export interface IgnoreEntry { reason: string; @@ -65,33 +62,29 @@ export const IGNORED_CASES: Readonly> = { "java-field-ref": { reason: "java: java module" }, "java-interop-enum": { reason: "java: java module" }, "java-interop-function-call": { reason: "java: java module" }, - "groupby-complex": { reason: "java: application/java content type" }, "write-function-with-null": { reason: "java: java module" }, "runtime_run_null_java": { reason: "java: java module" }, - // do-block cases previously skipped here now run: ensureOutputDirective - // splits on a column-0 `---`, so it no longer mis-detects the indented `---` - // inside a do {} block. Remaining try-handle-*/private_scope_directives cases - // fail for a different reason (unresolved module) and are grouped above. + // Previously-skipped cases that now run after ensureOutputDirective was + // strengthened: splitting on a column-0 `---` recovered the do-block cases + // (do-1, do-block, csv-streaming, range-selector, …), and replacing a + // conflicting output mime recovered format-mismatch cases (recursive_mapObject, + // xml-root-with-text, groupby-complex's application/java, tree-filter*, + // bad-inline, string_interpolation_selection, overload-functions, + // dynamic_attribute_name). Remaining try-handle-*/private_scope_directives + // fail on unresolved-module and are grouped above. - // output-directive-mismatch — transform pins a format ≠ expected extension - "recursive_mapObject": { reason: "output-directive-mismatch: transform outputs json, expected xml" }, - "xml-root-with-text": { reason: "output-directive-mismatch: transform pins a differing output" }, - "runtime_orElseTry": { reason: "output-directive-mismatch: transform pins a differing output" }, - "runtime_run": { reason: "output-directive-mismatch: transform pins a differing output" }, - "runtime_run_coercionException": { reason: "output-directive-mismatch: transform pins a differing output" }, - "runtime_run_fibo": { reason: "output-directive-mismatch: transform pins a differing output" }, - "try-recursive-call": { reason: "output-directive-mismatch: transform pins a differing output" }, - "runtime_dataFormatsDescriptors": { reason: "output-directive-mismatch: internal data-format descriptors" }, + // dw::Runtime module cases — the run/orElseTry/dataFormatsDescriptors behavior + // isn't available in this dwlib (not a directive issue). + "runtime_orElseTry": { reason: "dw::Runtime: run/orElseTry behavior not in dwlib" }, + "runtime_run": { reason: "dw::Runtime: run behavior not in dwlib" }, + "runtime_run_coercionException": { reason: "dw::Runtime: run behavior not in dwlib" }, + "runtime_run_fibo": { reason: "dw::Runtime: run behavior not in dwlib" }, + "try-recursive-call": { reason: "dw::Runtime: run behavior not in dwlib" }, + "runtime_dataFormatsDescriptors": { reason: "dw::Runtime: internal data-format descriptors" }, // dwl-output-format — application/dw formatting differs from expected DWL - "tree-filterArrayLeafs": { reason: "dwl-output-format: @(…) metadata / quoting differs" }, - "tree-filterObjectLeafs": { reason: "dwl-output-format: @(…) metadata / quoting differs" }, - "tree-filterTree": { reason: "dwl-output-format: @(…) metadata / quoting differs" }, - "bad-inline": { reason: "dwl-output-format: application/dw formatting differs" }, - "string_interpolation_selection": { reason: "dwl-output-format: unquoted keys in application/dw output" }, "coerciones_toString": { reason: "dwl-output-format: application/dw formatting differs" }, - "overload-functions": { reason: "dwl-output-format: application/dw formatting differs" }, // multipart — multipart writing edge cases "multipart-binary": { reason: "multipart: binary part comparison unsupported" }, @@ -116,7 +109,6 @@ export const IGNORED_CASES: Readonly> = { "xml-value-selector": { reason: "xml: namespace scoping in fixture" }, "xml-streaming-selectors": { reason: "xml: streaming selector serialization" }, "xml_empty_namespace": { reason: "xml: empty namespace serialization" }, - "dynamic_attribute_name": { reason: "xml: dynamic attribute serialization" }, }; /** Whether a case is on the ignore list. */ diff --git a/native-lib/node/tests/tck/transform.ts b/native-lib/node/tests/tck/transform.ts index aacf734..1e04c3f 100644 --- a/native-lib/node/tests/tck/transform.ts +++ b/native-lib/node/tests/tck/transform.ts @@ -4,28 +4,28 @@ // the output directive to the format implied by the expected file's extension. // We can't run that AST rewriter from Node, so we do a deliberately small // text-level normalization that covers the common cases and lean on the ignore -// list for the rest (e.g. `do`-block headers, which a regex can't safely -// rewrite). Empirically this runs ~91% of supported runtime cases. +// list for the rest. // // Rule: split on the document body separator — a `---` at column 0 — into -// header + body. If the header has no `output` directive, append one for the -// target MIME. Matching only column-0 `---` avoids mis-splitting on the -// indented `---` that appears inside a `do { … }` block (whose document -// separator, if any, is still at column 0). Transforms that pin a *conflicting* -// output format are a separate case handled via the ignore list, not here, -// since replacing a directive needs the AST rewrite the CLI uses. +// header + body (matching only column-0 avoids mis-splitting on the indented +// `---` inside a `do { … }` block), then: +// - no header `output` directive → append `output `; +// - header pins a plain `mime/type` of a DIFFERENT family than the target → +// replace just the mime token (keeping trailing options), matching what the +// CLI's AST rewriter does for the expected extension; +// - otherwise leave the directive alone. "Otherwise" deliberately covers +// same-family directives, `multipart/*` subtypes (so a pinned +// `multipart/mixed` with its `boundary=` option isn't clobbered), and +// non-mime output selectors like `output :Type json`. + +import { familyForMime } from "./formats"; /** - * Ensures the transform declares an `output` directive for `mime`. - * - * If the header (everything before the first column-0 `---`) already has an - * `output` line, the script is returned unchanged. Otherwise `output ` is - * appended to the header. A transform with no column-0 `---` is treated as a - * bare body and given a fresh `output \n---\n` header. + * Ensures the transform's `output` directive targets `mime`. * * @param src - The raw transform.dwl contents. * @param mime - The MIME type implied by the expected output file's extension. - * @returns The transform with a guaranteed output directive. + * @returns The transform with an output directive for the target format. */ export function ensureOutputDirective(src: string, mime: string): string { const lines = src.split(/\r?\n/); @@ -41,9 +41,19 @@ export function ensureOutputDirective(src: string, mime: string): string { const header = lines.slice(0, sep); const body = lines.slice(sep + 1); - if (header.some((l) => /^\s*output\s+/.test(l))) { - return src; // already has an output directive + const outputIdx = header.findIndex((l) => /^\s*output\s+/.test(l)); + if (outputIdx < 0) { + return [...header, `output ${mime}`].join("\n") + "\n---\n" + body.join("\n"); + } + + // Replace the mime token only when it is a plain `mime/type` of a different + // family than the target. Leave `:Type` selectors and same-/multipart-family + // directives untouched to avoid corrupting a directive that was correct. + const token = header[outputIdx].match(/^\s*output\s+(\S+)/)?.[1] ?? ""; + const targetFamily = familyForMime(mime); + if (token.includes("/") && familyForMime(token) !== targetFamily && familyForMime(token) !== "multipart") { + header[outputIdx] = header[outputIdx].replace(/^(\s*output\s+)\S+/, `$1${mime}`); } - return [...header, `output ${mime}`].join("\n") + "\n---\n" + body.join("\n"); + return header.join("\n") + "\n---\n" + body.join("\n"); } \ No newline at end of file diff --git a/native-lib/node/tests/unit/tck-transform.test.ts b/native-lib/node/tests/unit/tck-transform.test.ts index a53c525..86d9720 100644 --- a/native-lib/node/tests/unit/tck-transform.test.ts +++ b/native-lib/node/tests/unit/tck-transform.test.ts @@ -8,8 +8,30 @@ describe("ensureOutputDirective", () => { expect(out).toBe("type IName = String\noutput application/json\n---\n{ a: 1 }"); }); - it("leaves a transform with an existing output directive unchanged", () => { - const src = "%dw 2.0\noutput application/xml\n---\npayload"; + it("leaves an existing directive unchanged when it already targets the format", () => { + const src = "%dw 2.0\noutput application/json\n---\npayload"; + expect(ensureOutputDirective(src, "application/json")).toBe(src); + }); + + it("replaces the mime token when the pinned format differs from the target", () => { + const src = "%dw 2.0\noutput application/json\n---\npayload"; + expect(ensureOutputDirective(src, "application/xml")).toBe("%dw 2.0\noutput application/xml\n---\npayload"); + }); + + it("preserves trailing directive options when replacing the mime", () => { + const src = '%dw 2.0\noutput application/json encoding="UTF-16"\n---\npayload'; + expect(ensureOutputDirective(src, "text/plain")).toBe('%dw 2.0\noutput text/plain encoding="UTF-16"\n---\npayload'); + }); + + it("does not clobber a multipart subtype directive", () => { + // A pinned multipart/mixed (with its boundary option) must survive even + // though the expected extension maps to multipart/form-data. + const src = '%dw 2.0\noutput multipart/mixed boundary="abc"\n---\npayload'; + expect(ensureOutputDirective(src, "multipart/form-data")).toBe(src); + }); + + it("does not touch a non-mime output selector (output :Type json)", () => { + const src = "%dw 2.0\noutput :Test.number json\n---\npayload"; expect(ensureOutputDirective(src, "application/json")).toBe(src); });