diff --git a/doc/Benchmarking.md b/doc/Benchmarking.md new file mode 100644 index 0000000000..7f8e254452 --- /dev/null +++ b/doc/Benchmarking.md @@ -0,0 +1,91 @@ +# Pipeline performance benchmark + +Run the benchmark from the repository root: + +```bash +npm run benchmark:pipeline +``` + +The command builds `quicktype-core`, warms up every case, and reports median, +p95, and minimum end-to-end times plus maximum observed used heap. It covers +small and large JSON samples, small and large JSON Schemas, and the TypeScript +and Rust renderers. Inputs are generated deterministically in memory so +results do not include filesystem I/O. + +## Canonical real-world benchmark + +Run the canonical benchmark set with: + +```bash +npm run benchmark:canonical +``` + +This command builds `quicktype-core`, gives Node an 8 GiB heap, and runs one +warmup plus three measured passes for both TypeScript and Rust over exactly +these inputs: + +- USGS all-month earthquakes GeoJSON +- GitHub REST OpenAPI description +- NVD CVE 2024 feed +- HL7 FHIR R5 combined JSON Schema +- Kestra 0.19 JSON Schema + +The inputs are downloaded on the first run and cached outside the repository in +the operating system's user cache directory. Downloads, decompression, and +filesystem reads are not timed. Repeated runs use the same snapshots; download +current copies with: + +```bash +npm run benchmark:canonical -- --refresh +``` + +Set `QUICKTYPE_BENCHMARK_CACHE` or pass `--cache-dir DIR` to choose a different +cache location. The canonical command accepts the same `--warmup`, +`--iterations`, and `--json` options as the synthetic pipeline benchmark. + +The end-to-end timer begins before quicktype parses or registers the input and +ends after the generated source has been serialized. The phase breakdown is: + +- **Parse**: compressed JSON parsing for JSON samples, or YAML/JSON parsing for + JSON Schema. +- **Infer/schema**: type inference from compressed JSON, or conversion from the + parsed schema to quicktype's initial type graph. +- **Transform**: graph rewrites, map and enum inference, transformations, + garbage collection, and name gathering. +- **Codegen**: target-language rendering and serialization. +- **Other**: input setup, graph setup, callback overhead, and uninstrumented + control flow. + +The phase breakdown uses the run at the median end-to-end time. When the +iteration count is even, the two middle runs are interpolated. This keeps the +phases additive and their percentages at 100%. + +Individual pass timings are also printed. Pass timings are inclusive, and +repeated passes such as `flatten unions` are combined within each run. + +`Max heap` is the largest `process.memoryUsage().heapUsed` value observed across +the measured runs for a case. Heap is sampled before and after input parsing, +after every instrumented quicktype pass, after rendering, and after output +serialization. Node does not expose an exact JavaScript-heap high-water mark, +so allocations created and released entirely within one synchronous pass might +not appear in this value. The JSON output records it as +`memory.maximumHeapUsedBytes`. + +Use fewer samples for a quick smoke test or emit JSON for comparison tooling: + +```bash +npm run benchmark:pipeline -- --warmup 0 --iterations 1 +npm run benchmark:pipeline -- --iterations 10 --json > benchmark.json +``` + +For less garbage-collection noise, invoke the built benchmark directly with +Node's explicit GC enabled: + +```bash +npm run build --workspace quicktype-core +node --expose-gc script/benchmark-pipeline.mjs +``` + +Compare results only on the same machine, Node version, power mode, and command +line. The benchmark intentionally reports distributions rather than enforcing +a fixed pass/fail threshold. diff --git a/package.json b/package.json index 028a227ecb..09df9da7a1 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "test:release": "tsx test/release-version.ts", "benchmark:markov-chain": "npm run build --workspace quicktype-core && node --expose-gc script/benchmark-markov-chain.mjs", "benchmark:markov-chain:fidelity": "npm run build --workspace quicktype-core && node script/benchmark-markov-fidelity.mjs", + "benchmark:pipeline": "npm run build --workspace quicktype-core && node script/benchmark-pipeline.mjs", + "benchmark:canonical": "npm run build --workspace quicktype-core && node --expose-gc --max-old-space-size=8192 script/benchmark-pipeline.mjs --canonical --iterations 3", "start": "script/watch", "clean": "rm -rf dist *~ packages/*/{dist,out}", "debug": "node --inspect-brk --max-old-space-size=4096 ./dist/index.js", diff --git a/packages/quicktype-core/src/Naming.ts b/packages/quicktype-core/src/Naming.ts index 83a3a11b9c..008646ab7e 100644 --- a/packages/quicktype-core/src/Naming.ts +++ b/packages/quicktype-core/src/Naming.ts @@ -24,6 +24,8 @@ export class Namespace { private readonly _members = new Set(); + private _forbiddenNameds: ReadonlySet | undefined; + public constructor( _name: string, parent: Namespace | undefined, @@ -50,11 +52,14 @@ export class Namespace { } public get forbiddenNameds(): ReadonlySet { - // FIXME: cache - return setUnion( - this.additionalForbidden, - ...Array.from(this.forbiddenNamespaces).map((ns) => ns.members), - ); + if (this._forbiddenNameds === undefined) { + this._forbiddenNameds = setUnion( + this.additionalForbidden, + ...Array.from(this.forbiddenNamespaces).map((ns) => ns.members), + ); + } + + return this._forbiddenNameds; } public add(named: TName): TName { @@ -451,17 +456,23 @@ export function assignNames( } } + const unfinishedNamespaces = setFilter(ctx.namespaces, (ns) => + iterableSome(ns.members, (member) => !ctx.names.has(member)), + ); + for (;;) { // 1. Find a namespace whose forbiddens are all fully named, and which has // at least one unnamed Named that has all its dependencies satisfied. // If no such namespace exists we're either done, or there's an unallowed // cycle. - const unfinishedNamespaces = setFilter(ctx.namespaces, (ns) => - ctx.areForbiddensFullyNamed(ns), - ); - const readyNamespace = iterableFind(unfinishedNamespaces, (ns) => - iterableSome(ns.members, (member) => ctx.isReadyToBeNamed(member)), + const readyNamespace = iterableFind( + unfinishedNamespaces, + (ns) => + ctx.areForbiddensFullyNamed(ns) && + iterableSome(ns.members, (member) => + ctx.isReadyToBeNamed(member), + ), ); if (readyNamespace === undefined) { @@ -519,5 +530,13 @@ export function assignNames( } } } + + if ( + iterableEvery(readyNamespace.members, (member) => + ctx.names.has(member), + ) + ) { + unfinishedNamespaces.delete(readyNamespace); + } } } diff --git a/packages/quicktype-core/src/Run.ts b/packages/quicktype-core/src/Run.ts index 261cdec62f..898640aff5 100644 --- a/packages/quicktype-core/src/Run.ts +++ b/packages/quicktype-core/src/Run.ts @@ -108,6 +108,8 @@ export interface NonInferenceOptions { leadingComments?: Comment[]; /** Don't render output. This is mainly useful for benchmarking. */ noRender: boolean; + /** Called after each measured pipeline pass. This is mainly useful for benchmarking. */ + onTiming?: (timing: QuicktypeTiming) => void; /** Name of the output file. Note that quicktype will not write that file, but you'll get its name * back as a key in the resulting `Map`. */ @@ -116,6 +118,11 @@ export interface NonInferenceOptions { rendererOptions: Partial>; } +export interface QuicktypeTiming { + milliseconds: number; + name: string; +} + export type Options = NonInferenceOptions & InferenceFlags; @@ -126,6 +133,7 @@ const defaultOptions: NonInferenceOptions = { allPropertiesOptional: false, fixedTopLevels: false, noRender: false, + onTiming: undefined, leadingComments: undefined, rendererOptions: {}, indentation: undefined, @@ -195,24 +203,28 @@ class Run implements RunContext { } public async timeSync(name: string, f: () => Promise): Promise { - const start = Date.now(); + const start = performance.now(); const result = await f(); - const end = Date.now(); + const milliseconds = performance.now() - start; if (this._options.debugPrintTimes) { - console.log(`${name} took ${end - start}ms`); + console.log(`${name} took ${milliseconds.toFixed(3)}ms`); } + this._options.onTiming?.({ name, milliseconds }); + return result; } public time(name: string, f: () => T): T { - const start = Date.now(); + const start = performance.now(); const result = f(); - const end = Date.now(); + const milliseconds = performance.now() - start; if (this._options.debugPrintTimes) { - console.log(`${name} took ${end - start}ms`); + console.log(`${name} took ${milliseconds.toFixed(3)}ms`); } + this._options.onTiming?.({ name, milliseconds }); + return result; } @@ -580,18 +592,20 @@ class Run implements RunContext { targetLanguage: TargetLanguage, graph: TypeGraph, ): MultiFileRenderResult { - if (this._options.noRender) { - return this.makeSimpleTextResult(["Done.", ""]); - } + return this.time("render", () => { + if (this._options.noRender) { + return this.makeSimpleTextResult(["Done.", ""]); + } - return targetLanguage.renderGraphAndSerialize( - graph, - this._options.outputFilename, - this._options.alphabetizeProperties, - this._options.leadingComments, - this._options.rendererOptions, - this._options.indentation, - ); + return targetLanguage.renderGraphAndSerialize( + graph, + this._options.outputFilename, + this._options.alphabetizeProperties, + this._options.leadingComments, + this._options.rendererOptions, + this._options.indentation, + ); + }); } } diff --git a/packages/quicktype-core/src/UnionBuilder.ts b/packages/quicktype-core/src/UnionBuilder.ts index 1e9f0229bd..611a147858 100644 --- a/packages/quicktype-core/src/UnionBuilder.ts +++ b/packages/quicktype-core/src/UnionBuilder.ts @@ -78,6 +78,8 @@ function addAttributesToBuilder( if (arr === undefined) { arr = []; builder.set(kind, arr); + } else if (newAttributes.size === 0) { + return; } arr.push(newAttributes); @@ -108,6 +110,8 @@ export class UnionAccumulator private readonly _stringTypeAttributes: TypeAttributeMapBuilder = new Map(); + private readonly _stringCases = new Map(); + public readonly arrayData: TArray[] = []; public readonly objectData: TObject[] = []; @@ -251,7 +255,14 @@ export class UnionAccumulator } public addStringCases(cases: string[], attributes: TypeAttributes): void { - this.addFullStringType(attributes, StringTypes.fromCases(cases)); + addAttributesToBuilder( + this._stringTypeAttributes, + "string", + attributes, + ); + for (const s of new Set(cases)) { + this._stringCases.set(s, (this._stringCases.get(s) ?? 0) + 1); + } } public addStringCase( @@ -259,7 +270,12 @@ export class UnionAccumulator count: number, attributes: TypeAttributes, ): void { - this.addFullStringType(attributes, StringTypes.fromCase(s, count)); + addAttributesToBuilder( + this._stringTypeAttributes, + "string", + attributes, + ); + this._stringCases.set(s, (this._stringCases.get(s) ?? 0) + count); } public get enumCases(): ReadonlySet { @@ -272,9 +288,22 @@ export class UnionAccumulator "We can't have both strings and enums in the same union", ); + const stringTypeAttributes = buildTypeAttributeMap( + this._stringTypeAttributes, + ); + if (this._stringCases.size > 0) { + setAttributes( + stringTypeAttributes, + "string", + stringTypesTypeAttributeKind.makeAttributes( + new StringTypes(new Map(this._stringCases), new Set()), + ), + ); + } + const merged = mapMerge( buildTypeAttributeMap(this._nonStringTypeAttributes), - buildTypeAttributeMap(this._stringTypeAttributes), + stringTypeAttributes, ); if (merged.size === 0) { diff --git a/packages/quicktype-core/src/index.ts b/packages/quicktype-core/src/index.ts index 0b0089a13c..174f2df5af 100644 --- a/packages/quicktype-core/src/index.ts +++ b/packages/quicktype-core/src/index.ts @@ -1,5 +1,6 @@ export { type Options, + type QuicktypeTiming, getTargetLanguage, quicktypeMultiFile, quicktypeMultiFileSync, diff --git a/packages/quicktype-core/src/input/JSONSchemaInput.ts b/packages/quicktype-core/src/input/JSONSchemaInput.ts index 1fb8a32233..893893d0ad 100644 --- a/packages/quicktype-core/src/input/JSONSchemaInput.ts +++ b/packages/quicktype-core/src/input/JSONSchemaInput.ts @@ -1458,6 +1458,7 @@ async function refsInSchemaForURI( class InputJSONSchemaStore extends JSONSchemaStore { public constructor( private readonly _inputs: Map, + private readonly _ctx: RunContext, private readonly _delegate?: JSONSchemaStore, ) { super(); @@ -1466,9 +1467,11 @@ class InputJSONSchemaStore extends JSONSchemaStore { public async fetch(address: string): Promise { const maybeInput = this._inputs.get(address); if (maybeInput !== undefined) { - return checkJSONSchema( - parseJSON(maybeInput, "JSON Schema", address), - () => Ref.root(address), + return this._ctx.time("parse JSON Schema", () => + checkJSONSchema( + parseJSON(maybeInput, "JSON Schema", address), + () => Ref.root(address), + ), ); } @@ -1540,6 +1543,7 @@ export class JSONSchemaInput implements Input { } else { this._schemaStore = new InputJSONSchemaStore( this._schemaInputs, + ctx, maybeSchemaStore, ); maybeSchemaStore = this._schemaStore; diff --git a/packages/quicktype-core/src/support/ParseJSON.ts b/packages/quicktype-core/src/support/ParseJSON.ts index 3737dde66b..f7724ce939 100644 --- a/packages/quicktype-core/src/support/ParseJSON.ts +++ b/packages/quicktype-core/src/support/ParseJSON.ts @@ -14,7 +14,11 @@ export function parseJSON( text = text.slice(1); } - return YAML.parse(text); + try { + return JSON.parse(text); + } catch { + return YAML.parse(text); + } } catch (e) { let message: string; diff --git a/script/benchmark-pipeline.mjs b/script/benchmark-pipeline.mjs new file mode 100644 index 0000000000..42bda04d58 --- /dev/null +++ b/script/benchmark-pipeline.mjs @@ -0,0 +1,658 @@ +#!/usr/bin/env node + +import { createWriteStream } from "node:fs"; +import { mkdir, readFile, rename, rm, stat } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; + +import { + InputData, + JSONSchemaInput, + jsonInputForTargetLanguage, + quicktype, +} from "../packages/quicktype-core/dist/index.js"; + +const DEFAULT_ITERATIONS = 5; +const DEFAULT_WARMUPS = 1; +const LARGE_JSON_RECORDS = 5_000; +const LARGE_SCHEMA_TYPES = 180; +const languages = ["typescript", "rust"]; + +const canonicalInputDefinitions = [ + { + filename: "usgs-month.geojson", + kind: "json", + minimumBytes: 500_000, + name: "USGS earthquakes", + url: "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_month.geojson", + }, + { + filename: "github-openapi.json", + kind: "json", + minimumBytes: 5_000_000, + name: "GitHub REST OpenAPI", + url: "https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.json", + }, + { + compression: "gzip", + filename: "nvd-2024.json", + kind: "json", + minimumBytes: 100_000_000, + name: "NVD CVE 2024", + url: "https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2024.json.gz", + }, + { + filename: "fhir.schema.json", + kind: "schema", + minimumBytes: 3_000_000, + name: "HL7 FHIR R5", + url: "https://hl7.org/fhir/R5/fhir.schema.json", + }, + { + filename: "kestra.schema.json", + kind: "schema", + minimumBytes: 5_000_000, + name: "Kestra 0.19", + url: "https://hostr.flingit.run/s/quicktype-repros/kestra-0.19.0.schema.json", + }, +]; + +function defaultCacheDirectory() { + const platformCache = + process.platform === "win32" + ? process.env.LOCALAPPDATA + : (process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache")); + return join(platformCache ?? tmpdir(), "quicktype", "canonical-benchmark"); +} + +function usage() { + console.log(`Usage: npm run benchmark:pipeline -- [options] + +Options: + --iterations N Measured runs per case (default: ${DEFAULT_ITERATIONS}) + --warmup N Warmup runs per case (default: ${DEFAULT_WARMUPS}) + --canonical Run the five canonical real-world inputs + --cache-dir DIR Cache canonical inputs in DIR + --refresh Download fresh canonical input snapshots + --json Emit machine-readable JSON instead of tables + --help Show this help +`); +} + +function positiveInteger(value, option) { + const parsed = Number.parseInt(value, 10); + if (!Number.isSafeInteger(parsed) || parsed < 1) { + throw new Error(`${option} must be a positive integer`); + } + return parsed; +} + +function nonnegativeInteger(value, option) { + const parsed = Number.parseInt(value, 10); + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new Error(`${option} must be a nonnegative integer`); + } + return parsed; +} + +function parseArguments(args) { + const options = { + cacheDirectory: + process.env.QUICKTYPE_BENCHMARK_CACHE ?? defaultCacheDirectory(), + canonical: false, + iterations: DEFAULT_ITERATIONS, + json: false, + refresh: false, + warmups: DEFAULT_WARMUPS, + }; + + let i = 0; + while (i < args.length) { + const argument = args[i]; + switch (argument) { + case "--iterations": + options.iterations = positiveInteger( + args[i + 1], + "--iterations", + ); + i += 2; + break; + case "--warmup": + options.warmups = nonnegativeInteger(args[i + 1], "--warmup"); + i += 2; + break; + case "--json": + options.json = true; + i++; + break; + case "--canonical": + options.canonical = true; + i++; + break; + case "--cache-dir": + if (args[i + 1] === undefined) { + throw new Error("--cache-dir requires a directory"); + } + options.cacheDirectory = args[i + 1]; + i += 2; + break; + case "--refresh": + options.refresh = true; + i++; + break; + case "--help": + usage(); + process.exit(0); + break; + default: + throw new Error(`Unknown option: ${argument}`); + } + } + + return options; +} + +function makeRecord(index) { + const record = { + id: `00000000-0000-4000-8000-${index.toString().padStart(12, "0")}`, + createdAt: new Date(1_700_000_000_000 + index * 60_000).toISOString(), + status: ["queued", "running", "complete", "failed"][index % 4], + score: index % 7 === 0 ? index + 0.25 : index, + tags: [`group-${index % 20}`, `region-${index % 8}`], + owner: { + id: index % 100, + name: `Owner ${index % 100}`, + profile: `https://example.com/owners/${index % 100}`, + }, + metrics: { + attempts: index % 5, + durationMs: index * 3.5, + successful: index % 9 !== 0, + }, + }; + + if (index % 3 === 0) { + record.note = `Generated benchmark record ${index}`; + } + if (index % 5 === 0) { + record.metadata = { source: "benchmark", shard: index % 16 }; + } + + return record; +} + +function makeJSON(recordCount) { + return JSON.stringify({ + generatedAt: "2026-01-01T00:00:00Z", + records: Array.from({ length: recordCount }, (_, index) => + makeRecord(index), + ), + }); +} + +function makeEntitySchema(index) { + const properties = { + id: { format: "uuid", type: "string" }, + createdAt: { format: "date-time", type: "string" }, + status: { + enum: ["queued", "running", "complete", "failed"], + type: "string", + }, + score: { minimum: 0, type: "number" }, + tags: { items: { type: "string" }, type: "array" }, + metadata: { + additionalProperties: { type: "string" }, + type: "object", + }, + }; + + if (index > 0) { + properties.parent = { $ref: `#/definitions/Entity${index - 1}` }; + } + + return { + additionalProperties: false, + properties, + required: ["id", "createdAt", "status", "score", "tags"], + title: `Entity ${index}`, + type: "object", + }; +} + +function makeSchema(typeCount) { + const definitions = Object.fromEntries( + Array.from({ length: typeCount }, (_, index) => [ + `Entity${index}`, + makeEntitySchema(index), + ]), + ); + const properties = Object.fromEntries( + Array.from({ length: typeCount }, (_, index) => [ + `entity${index}`, + { $ref: `#/definitions/Entity${index}` }, + ]), + ); + + return JSON.stringify({ + $schema: "http://json-schema.org/draft-07/schema#", + additionalProperties: false, + definitions, + properties, + required: Object.keys(properties), + title: "Benchmark", + type: "object", + }); +} + +const inputs = [ + { kind: "json", size: "small", source: makeJSON(4) }, + { kind: "json", size: "large", source: makeJSON(LARGE_JSON_RECORDS) }, + { kind: "schema", size: "small", source: makeSchema(3) }, + { kind: "schema", size: "large", source: makeSchema(LARGE_SCHEMA_TYPES) }, +]; + +async function cachedInputIsUsable(filename, minimumBytes) { + try { + return (await stat(filename)).size >= minimumBytes; + } catch (error) { + if ( + error instanceof Error && + "code" in error && + error.code === "ENOENT" + ) { + return false; + } + throw error; + } +} + +async function downloadCanonicalInput(definition, filename) { + console.error(`Downloading ${definition.name} from ${definition.url}`); + const response = await fetch(definition.url, { + headers: { "user-agent": "quicktype-canonical-benchmark" }, + }); + if (!response.ok || response.body === null) { + throw new Error( + `Could not download ${definition.name}: HTTP ${response.status}`, + ); + } + + const temporaryFilename = `${filename}.${process.pid}.tmp`; + await rm(temporaryFilename, { force: true }); + try { + const transforms = + definition.compression === "gzip" ? [createGunzip()] : []; + await pipeline( + Readable.fromWeb(response.body), + ...transforms, + createWriteStream(temporaryFilename), + ); + if ( + !(await cachedInputIsUsable( + temporaryFilename, + definition.minimumBytes, + )) + ) { + throw new Error( + `Downloaded ${definition.name} is unexpectedly small`, + ); + } + await rm(filename, { force: true }); + await rename(temporaryFilename, filename); + } catch (error) { + await rm(temporaryFilename, { force: true }); + throw error; + } +} + +async function loadCanonicalInput(definition, options) { + await mkdir(options.cacheDirectory, { recursive: true }); + const filename = join(options.cacheDirectory, definition.filename); + if ( + options.refresh || + !(await cachedInputIsUsable(filename, definition.minimumBytes)) + ) { + await downloadCanonicalInput(definition, filename); + } else { + console.error(`Using cached ${definition.name}: ${filename}`); + } + + return { + kind: definition.kind, + name: definition.name, + size: "canonical", + source: await readFile(filename, "utf8"), + sourceURL: definition.url, + }; +} + +function sumEvents(events, name) { + return events + .filter((event) => event.name === name) + .reduce((sum, event) => sum + event.milliseconds, 0); +} + +async function runOnce(input, language) { + const events = []; + const inputData = new InputData(); + const started = performance.now(); + let jsonParseMilliseconds = 0; + let maximumHeapUsedBytes = 0; + const observeHeap = () => { + maximumHeapUsedBytes = Math.max( + maximumHeapUsedBytes, + process.memoryUsage().heapUsed, + ); + }; + + observeHeap(); + + if (input.kind === "json") { + const jsonInput = jsonInputForTargetLanguage(language); + const parseStarted = performance.now(); + await jsonInput.addSource({ + name: "Benchmark", + samples: [input.source], + }); + jsonParseMilliseconds = performance.now() - parseStarted; + inputData.addInput(jsonInput); + observeHeap(); + } else { + const schemaInput = new JSONSchemaInput(undefined); + await schemaInput.addSource({ + name: "Benchmark", + schema: input.source, + }); + inputData.addInput(schemaInput); + } + + const output = await quicktype({ + inputData, + lang: language, + onTiming: (timing) => { + events.push(timing); + observeHeap(); + }, + }); + observeHeap(); + const totalMilliseconds = performance.now() - started; + const schemaParseMilliseconds = sumEvents(events, "parse JSON Schema"); + const readInputMilliseconds = sumEvents(events, "read input"); + const renderMilliseconds = sumEvents(events, "render"); + const transformationMilliseconds = events + .filter( + (event) => + event.name !== "read input" && + event.name !== "render" && + event.name !== "parse JSON Schema" && + !event.name.startsWith(" "), + ) + .reduce((sum, event) => sum + event.milliseconds, 0); + const parsingMilliseconds = + input.kind === "json" ? jsonParseMilliseconds : schemaParseMilliseconds; + const inferenceMilliseconds = Math.max( + 0, + readInputMilliseconds - schemaParseMilliseconds, + ); + const accountedMilliseconds = + parsingMilliseconds + + inferenceMilliseconds + + transformationMilliseconds + + renderMilliseconds; + + const passes = new Map(); + if (input.kind === "json") { + passes.set("parse JSON input", jsonParseMilliseconds); + } + for (const event of events) { + passes.set( + event.name, + (passes.get(event.name) ?? 0) + event.milliseconds, + ); + } + + const serializedOutput = output.lines.join("\n"); + const outputBytes = Buffer.byteLength(serializedOutput); + observeHeap(); + + return { + maximumHeapUsedBytes, + outputBytes, + passes: Object.fromEntries(passes), + phases: { + codeGeneration: renderMilliseconds, + inference: inferenceMilliseconds, + other: Math.max(0, totalMilliseconds - accountedMilliseconds), + parsing: parsingMilliseconds, + transformations: transformationMilliseconds, + }, + totalMilliseconds, + }; +} + +function percentile(values, percentileValue) { + const sorted = values.toSorted((a, b) => a - b); + const index = Math.max( + 0, + Math.ceil((percentileValue / 100) * sorted.length) - 1, + ); + return sorted[index]; +} + +function median(values) { + const sorted = values.toSorted((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +} + +function summarizeSamples(input, language, samples) { + const totalValues = samples.map((sample) => sample.totalMilliseconds); + const samplesByTotal = samples.toSorted( + (left, right) => left.totalMilliseconds - right.totalMilliseconds, + ); + const middle = Math.floor(samplesByTotal.length / 2); + const phaseNames = Object.keys(samples[0].phases); + const passNames = new Set( + samples.flatMap((sample) => Object.keys(sample.passes)), + ); + const phases = Object.fromEntries( + phaseNames.map((name) => { + const milliseconds = + samplesByTotal.length % 2 === 0 + ? (samplesByTotal[middle - 1].phases[name] + + samplesByTotal[middle].phases[name]) / + 2 + : samplesByTotal[middle].phases[name]; + return [name, milliseconds]; + }), + ); + const passes = Object.fromEntries( + Array.from(passNames, (name) => [ + name, + median(samples.map((sample) => sample.passes[name] ?? 0)), + ]), + ); + + return { + case: `${input.name ?? `${input.kind}/${input.size}`}/${language}`, + input: { + bytes: Buffer.byteLength(input.source), + kind: input.kind, + name: input.name, + size: input.size, + sourceURL: input.sourceURL, + }, + language, + memory: { + maximumHeapUsedBytes: Math.max( + ...samples.map((sample) => sample.maximumHeapUsedBytes), + ), + }, + outputBytes: median(samples.map((sample) => sample.outputBytes)), + passes, + phases, + total: { + medianMilliseconds: median(totalValues), + minMilliseconds: Math.min(...totalValues), + p95Milliseconds: percentile(totalValues, 95), + }, + }; +} + +function formatBytes(bytes) { + if (bytes >= 1_048_576) return `${(bytes / 1_048_576).toFixed(1)} MiB`; + if (bytes >= 1_024) return `${(bytes / 1_024).toFixed(1)} KiB`; + return `${bytes} B`; +} + +function formatMilliseconds(milliseconds) { + if (milliseconds >= 100) return milliseconds.toFixed(1); + if (milliseconds >= 10) return milliseconds.toFixed(2); + return milliseconds.toFixed(3); +} + +function table(headers, rows) { + const widths = headers.map((header, index) => + Math.max(header.length, ...rows.map((row) => row[index].length)), + ); + const formatRow = (row) => + row.map((cell, index) => cell.padEnd(widths[index])).join(" "); + return [ + formatRow(headers), + formatRow(widths.map((width) => "-".repeat(width))), + ...rows.map(formatRow), + ].join("\n"); +} + +function phaseCell(result, phase) { + const milliseconds = result.phases[phase]; + const percent = (milliseconds / result.total.medianMilliseconds) * 100; + return `${formatMilliseconds(milliseconds)} (${percent.toFixed(0)}%)`; +} + +function printResults(results, options) { + console.log( + `quicktype ${options.canonical ? "canonical " : ""}pipeline benchmark (${process.version}, ${process.platform}/${process.arch})`, + ); + console.log( + `${options.iterations} measured run(s), ${options.warmups} warmup(s) per case`, + ); + console.log(""); + console.log("End-to-end results (milliseconds)"); + console.log( + table( + ["Case", "Input", "Output", "Max heap", "Median", "p95", "Min"], + results.map((result) => [ + result.case, + formatBytes(result.input.bytes), + formatBytes(result.outputBytes), + formatBytes(result.memory.maximumHeapUsedBytes), + formatMilliseconds(result.total.medianMilliseconds), + formatMilliseconds(result.total.p95Milliseconds), + formatMilliseconds(result.total.minMilliseconds), + ]), + ), + ); + console.log(""); + console.log("Phase breakdown at median end-to-end time: milliseconds (%)"); + console.log( + table( + ["Case", "Parse", "Infer/schema", "Transform", "Codegen", "Other"], + results.map((result) => [ + result.case, + phaseCell(result, "parsing"), + phaseCell(result, "inference"), + phaseCell(result, "transformations"), + phaseCell(result, "codeGeneration"), + phaseCell(result, "other"), + ]), + ), + ); + console.log(""); + console.log("Hottest measured passes (median, inclusive)"); + for (const result of results) { + const hottest = Object.entries(result.passes) + .toSorted((left, right) => right[1] - left[1]) + .slice(0, 4) + .map( + ([name, milliseconds]) => + `${name} ${formatMilliseconds(milliseconds)} ms`, + ) + .join(", "); + console.log(` ${result.case}: ${hottest}`); + } +} + +async function main() { + const options = parseArguments(process.argv.slice(2)); + const results = []; + const selectedInputs = options.canonical + ? canonicalInputDefinitions + : inputs; + + for (const selectedInput of selectedInputs) { + const input = options.canonical + ? await loadCanonicalInput(selectedInput, options) + : selectedInput; + for (const language of languages) { + globalThis.gc?.(); + if (options.canonical) { + console.error(`Benchmarking ${input.name} -> ${language}`); + } + for (let i = 0; i < options.warmups; i++) { + await runOnce(input, language); + } + + const samples = []; + for (let i = 0; i < options.iterations; i++) { + globalThis.gc?.(); + samples.push(await runOnce(input, language)); + } + results.push(summarizeSamples(input, language, samples)); + } + if (options.canonical) { + input.source = ""; + globalThis.gc?.(); + } + } + + if (options.json) { + console.log( + JSON.stringify( + { + config: { + cacheDirectory: options.canonical + ? options.cacheDirectory + : undefined, + iterations: options.iterations, + largeJSONRecords: LARGE_JSON_RECORDS, + largeSchemaTypes: LARGE_SCHEMA_TYPES, + suite: options.canonical ? "canonical" : "synthetic", + warmups: options.warmups, + }, + platform: { + arch: process.arch, + node: process.version, + os: process.platform, + }, + results, + }, + undefined, + 2, + ), + ); + } else { + printResults(results, options); + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/test/unit/naming.test.ts b/test/unit/naming.test.ts new file mode 100644 index 0000000000..8023b656c6 --- /dev/null +++ b/test/unit/naming.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; + +import { + DependencyName, + Namer, + Namespace, + SimpleName, + assignNames, + keywordNamespace, +} from "../../packages/quicktype-core/src/Naming.js"; + +const identityNamer = new Namer("identity", (name) => name, ["renamed"]); + +describe("assignNames", () => { + it("revisits an unfinished namespace after a dependency is named", () => { + const firstNamespace = new Namespace("first", undefined, [], []); + const secondNamespace = new Namespace("second", undefined, [], []); + const first = firstNamespace.add( + new SimpleName(["first"], identityNamer, 1), + ); + const dependency = secondNamespace.add( + new SimpleName(["dependency"], identityNamer, 1), + ); + const dependent = firstNamespace.add( + new DependencyName( + identityNamer, + 2, + (lookup) => `${lookup(dependency)}Dependent`, + ), + ); + + const names = assignNames([firstNamespace, secondNamespace]); + + expect(names.get(first)).toBe("first"); + expect(names.get(dependency)).toBe("dependency"); + expect(names.get(dependent)).toBe("dependencyDependent"); + }); + + it("caches and observes forbidden namespace members", () => { + const reserved = keywordNamespace("reserved", ["match"]); + const namespace = new Namespace("names", undefined, [reserved], []); + const conflicting = namespace.add( + new SimpleName(["match"], identityNamer, 1), + ); + + const forbidden = namespace.forbiddenNameds; + expect(namespace.forbiddenNameds).toBe(forbidden); + expect(forbidden).toEqual(reserved.members); + + const names = assignNames([reserved, namespace]); + expect(names.get(conflicting)).toBe("renamed_match"); + }); +}); diff --git a/test/unit/parse-json.test.ts b/test/unit/parse-json.test.ts new file mode 100644 index 0000000000..b91848b073 --- /dev/null +++ b/test/unit/parse-json.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; + +import { parseJSON } from "quicktype-core"; + +describe("parseJSON", () => { + it("parses strict JSON", () => { + expect(parseJSON('{"name":"quicktype","count":2}', "test")).toEqual({ + count: 2, + name: "quicktype", + }); + }); + + it("falls back to YAML", () => { + expect(parseJSON("name: quicktype\ncount: 2\n", "test")).toEqual({ + count: 2, + name: "quicktype", + }); + }); + + it("accepts a byte-order mark before strict JSON", () => { + expect(parseJSON('\ufeff{"name":"quicktype"}', "test")).toEqual({ + name: "quicktype", + }); + }); +}); diff --git a/test/unit/pipeline-timing.test.ts b/test/unit/pipeline-timing.test.ts new file mode 100644 index 0000000000..a9dda538e5 --- /dev/null +++ b/test/unit/pipeline-timing.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; + +import { + InputData, + JSONSchemaInput, + type QuicktypeTiming, + jsonInputForTargetLanguage, + quicktype, +} from "quicktype-core"; + +describe("pipeline timing", () => { + it("reports input processing and rendering", async () => { + const jsonInput = jsonInputForTargetLanguage("typescript"); + await jsonInput.addSource({ + name: "Example", + samples: ['{"id": 1, "name": "quicktype"}'], + }); + const inputData = new InputData(); + inputData.addInput(jsonInput); + const timings: QuicktypeTiming[] = []; + + await quicktype({ + inputData, + lang: "typescript", + onTiming: (timing) => timings.push(timing), + }); + + expect(timings.some(({ name }) => name === "read input")).toBe(true); + expect(timings.some(({ name }) => name === "render")).toBe(true); + expect(timings.every(({ milliseconds }) => milliseconds >= 0)).toBe( + true, + ); + }); + + it("reports JSON Schema parsing separately", async () => { + const schemaInput = new JSONSchemaInput(undefined); + await schemaInput.addSource({ + name: "Example", + schema: JSON.stringify({ + properties: { id: { type: "integer" } }, + type: "object", + }), + }); + const inputData = new InputData(); + inputData.addInput(schemaInput); + const names: string[] = []; + + await quicktype({ + inputData, + lang: "typescript", + onTiming: ({ name }) => names.push(name), + }); + + expect(names).toContain("parse JSON Schema"); + expect(names).toContain("read input"); + expect(names).toContain("render"); + }); +}); diff --git a/test/unit/union-accumulator.test.ts b/test/unit/union-accumulator.test.ts new file mode 100644 index 0000000000..285ea9acfa --- /dev/null +++ b/test/unit/union-accumulator.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; + +import { stringTypesTypeAttributeKind } from "../../packages/quicktype-core/src/attributes/StringTypes.js"; +import { emptyTypeAttributes } from "../../packages/quicktype-core/src/attributes/TypeAttributes.js"; +import { defined } from "../../packages/quicktype-core/src/support/Support.js"; +import { UnionAccumulator } from "../../packages/quicktype-core/src/UnionBuilder.js"; + +describe("UnionAccumulator string cases", () => { + it("combines case counts without counting duplicates within one batch", () => { + const accumulator = new UnionAccumulator(true); + accumulator.addStringCase("red", 2, emptyTypeAttributes); + accumulator.addStringCases(["blue", "blue"], emptyTypeAttributes); + + const attributes = defined(accumulator.getMemberKinds().get("string")); + const stringTypes = defined( + stringTypesTypeAttributeKind.tryGetInAttributes(attributes), + ); + + expect(Array.from(defined(stringTypes.cases))).toEqual([ + ["red", 2], + ["blue", 1], + ]); + }); + + it("keeps an unrestricted string unrestricted", () => { + const accumulator = new UnionAccumulator(true); + accumulator.addStringCase("red", 1, emptyTypeAttributes); + accumulator.addStringType("string", emptyTypeAttributes); + + const attributes = defined(accumulator.getMemberKinds().get("string")); + const stringTypes = defined( + stringTypesTypeAttributeKind.tryGetInAttributes(attributes), + ); + + expect(stringTypes.cases).toBeUndefined(); + }); +});