diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index e1e30b95a5..0e53ae7b11 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -81,6 +81,8 @@ jobs: # Kotlin is also slow - fixture: kotlin,schema-kotlin,kotlin-jackson,schema-kotlin-jackson runs-on: ubuntu-latest-16-cores + - fixture: kotlinx,schema-kotlinx + runs-on: ubuntu-latest-16-cores # - fixture: objective-c # segfault on compiled test cmd # runs-on: macos-latest diff --git a/README.md b/README.md index 4bedfc5989..7cb3513fce 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![](https://raw.githubusercontent.com/quicktype/quicktype/master/media/quicktype-logo.svg?sanitize=true) [![npm version](https://badge.fury.io/js/quicktype.svg)](https://badge.fury.io/js/quicktype) -![Build status](https://github.com/quicktype/quicktype/actions/workflows/master.yaml/badge.svg) +[![Build status](https://github.com/glideapps/quicktype/actions/workflows/test-pr.yaml/badge.svg?branch=master)](https://github.com/glideapps/quicktype/actions/workflows/test-pr.yaml) `quicktype` generates strongly-typed models and serializers from JSON, JSON Schema, TypeScript, and [GraphQL queries](https://blog.quicktype.io/graphql-with-quicktype/), making it a breeze to work with JSON type-safely in many programming languages. diff --git a/packages/quicktype-core/src/input/JSONSchemaInput.ts b/packages/quicktype-core/src/input/JSONSchemaInput.ts index 893893d0ad..d3a0e4feef 100644 --- a/packages/quicktype-core/src/input/JSONSchemaInput.ts +++ b/packages/quicktype-core/src/input/JSONSchemaInput.ts @@ -1034,17 +1034,48 @@ async function addTypesInSchema( async function makeArrayType(): Promise { const singularAttributes = singularizeTypeNames(typeAttributes); const items = schema.items; + // JSON Schema 2020-12 renamed the array (tuple) form of `items` to + // `prefixItems`; treat it the same as a draft-07 array-valued + // `items` so tuple schemas produced by e.g. Pydantic v2 and + // schemars are not silently dropped. + const prefixItems = schema.prefixItems; + const tupleItems = Array.isArray(prefixItems) ? prefixItems : items; + const tupleKey = Array.isArray(prefixItems) + ? "prefixItems" + : "items"; let itemType: TypeRef; - if (Array.isArray(items)) { - const itemsLoc = loc.push("items"); - const itemTypes = await arrayMapSync(items, async (item, i) => { - const itemLoc = itemsLoc.push(i.toString()); - return await toType( - checkJSONSchema(item, itemLoc.canonicalRef), - itemLoc, - singularAttributes, + if (Array.isArray(tupleItems)) { + const itemsLoc = loc.push(tupleKey); + const itemTypes = await arrayMapSync( + tupleItems, + async (item, i) => { + const itemLoc = itemsLoc.push(i.toString()); + return await toType( + checkJSONSchema(item, itemLoc.canonicalRef), + itemLoc, + singularAttributes, + ); + }, + ); + // In 2020-12 an object-form `items` next to `prefixItems` + // describes the rest elements of an open tuple. quicktype + // models tuples as arrays of a union of the member types, so + // the rest type joins that union. A boolean `items` (`false` + // closes the tuple, `true` allows anything) is ignored. + if ( + tupleKey === "prefixItems" && + typeof items === "object" && + !Array.isArray(items) + ) { + const restItemsLoc = loc.push("items"); + itemTypes.push( + await toType( + checkJSONSchema(items, restItemsLoc.canonicalRef), + restItemsLoc, + singularAttributes, + ), ); - }); + } itemType = typeBuilder.getUnionType( emptyTypeAttributes, new Set(itemTypes), @@ -1229,6 +1260,7 @@ async function addTypesInSchema( schema.properties !== undefined || schema.additionalProperties !== undefined || schema.items !== undefined || + schema.prefixItems !== undefined || schema.required !== undefined || enumArray !== undefined || isConst; diff --git a/packages/quicktype-core/src/language/JavaScript/JavaScriptRenderer.ts b/packages/quicktype-core/src/language/JavaScript/JavaScriptRenderer.ts index f03e41f27f..0adcaefd30 100644 --- a/packages/quicktype-core/src/language/JavaScript/JavaScriptRenderer.ts +++ b/packages/quicktype-core/src/language/JavaScript/JavaScriptRenderer.ts @@ -493,15 +493,27 @@ function r(name${stringAnnotation}) { protected emitTypes(): void {} - protected emitUsageImportComment(): void { - this.emitLine('// const Convert = require("./file");'); + protected usageModuleName(givenOutputFilename: string): string { + return givenOutputFilename === "stdout" + ? "file" + : givenOutputFilename + .replace(/^.*[/\\]/, "") + .replace(/\.[^.]+$/, ""); + } + + protected emitUsageImportComment(givenOutputFilename: string): void { + this.emitLine( + '// const Convert = require("./', + this.usageModuleName(givenOutputFilename), + '");', + ); } - protected emitUsageComments(): void { + protected emitUsageComments(givenOutputFilename: string): void { this.emitMultiline(`// To parse this data: //`); - this.emitUsageImportComment(); + this.emitUsageImportComment(givenOutputFilename); this.emitLine("//"); this.forEachTopLevel("none", (_t, name) => { const camelCaseName = modifySource(camelCase, name); @@ -537,11 +549,11 @@ function r(name${stringAnnotation}) { }); } - protected emitSourceStructure(): void { + protected emitSourceStructure(givenOutputFilename: string): void { if (this.leadingComments !== undefined) { this.emitComments(this.leadingComments); } else { - this.emitUsageComments(); + this.emitUsageComments(givenOutputFilename); } this.emitTypes(); diff --git a/packages/quicktype-core/src/language/TypeScriptFlow/FlowRenderer.ts b/packages/quicktype-core/src/language/TypeScriptFlow/FlowRenderer.ts index 76446d9b34..a0a3bfaee2 100644 --- a/packages/quicktype-core/src/language/TypeScriptFlow/FlowRenderer.ts +++ b/packages/quicktype-core/src/language/TypeScriptFlow/FlowRenderer.ts @@ -39,9 +39,9 @@ export class FlowRenderer extends TypeScriptFlowBaseRenderer { }); } - protected emitSourceStructure(): void { + protected emitSourceStructure(givenOutputFilename: string): void { this.emitLine("// @flow"); this.ensureBlankLine(); - super.emitSourceStructure(); + super.emitSourceStructure(givenOutputFilename); } } diff --git a/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptFlowBaseRenderer.ts b/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptFlowBaseRenderer.ts index d4c2393d38..d20f56c19c 100644 --- a/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptFlowBaseRenderer.ts +++ b/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptFlowBaseRenderer.ts @@ -191,9 +191,9 @@ export abstract class TypeScriptFlowBaseRenderer extends JavaScriptRenderer { ); } - protected emitUsageComments(): void { + protected emitUsageComments(givenOutputFilename: string): void { if (this._tsFlowOptions.justTypes) return; - super.emitUsageComments(); + super.emitUsageComments(givenOutputFilename); } protected deserializerFunctionLine(t: Type, name: Name): Sourcelike { diff --git a/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptRenderer.ts b/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptRenderer.ts index 43537b1464..376036727d 100644 --- a/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptRenderer.ts +++ b/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptRenderer.ts @@ -50,7 +50,7 @@ export class TypeScriptRenderer extends TypeScriptFlowBaseRenderer { protected emitModuleExports(): void {} - protected emitUsageImportComment(): void { + protected emitUsageImportComment(givenOutputFilename: string): void { const topLevelNames: Sourcelike[] = []; this.forEachTopLevel( "none", @@ -62,7 +62,9 @@ export class TypeScriptRenderer extends TypeScriptFlowBaseRenderer { this.emitLine( "// import { Convert", topLevelNames, - ' } from "./file";', + ' } from "./', + this.usageModuleName(givenOutputFilename), + '";', ); } diff --git a/test/fixtures.ts b/test/fixtures.ts index d9f78eaea2..d8bc01ed3f 100644 --- a/test/fixtures.ts +++ b/test/fixtures.ts @@ -1562,6 +1562,7 @@ export const allFixtures: Fixture[] = [ new JSONFixture(languages.KotlinLanguage), new JSONFixture(languages.Scala3Language), new JSONFixture(languages.KotlinJacksonLanguage, "kotlin-jackson"), + new JSONFixture(languages.KotlinXLanguage, "kotlinx"), new JSONFixture(languages.DartLanguage), new JSONFixture(languages.PikeLanguage), new JSONFixture(languages.HaskellLanguage), @@ -1601,6 +1602,7 @@ export const allFixtures: Fixture[] = [ languages.KotlinJacksonLanguage, "schema-kotlin-jackson", ), + new JSONSchemaFixture(languages.KotlinXLanguage, "schema-kotlinx"), new JSONSchemaFixture(languages.Scala3Language), new JSONSchemaFixture(languages.DartLanguage), new JSONSchemaFixture(languages.PikeLanguage), diff --git a/test/fixtures/kotlinx/build.sh b/test/fixtures/kotlinx/build.sh new file mode 100755 index 0000000000..fc1ba9881c --- /dev/null +++ b/test/fixtures/kotlinx/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# The kotlinx-serialization compiler plugin ships with the Kotlin compiler +# distribution, in lib/ next to bin/kotlinc. +KOTLINC_PATH="$(readlink -f "$(command -v kotlinc)")" +KOTLIN_LIB="$(cd "$(dirname "$KOTLINC_PATH")/../lib" && pwd)" + +kotlinc main.kt TopLevel.kt -include-runtime -Xplugin="$KOTLIN_LIB/kotlinx-serialization-compiler-plugin.jar" -cp kotlinx-serialization-core-jvm-1.7.3.jar:kotlinx-serialization-json-jvm-1.7.3.jar -d main.jar diff --git a/test/fixtures/kotlinx/kotlinx-serialization-core-jvm-1.7.3.jar b/test/fixtures/kotlinx/kotlinx-serialization-core-jvm-1.7.3.jar new file mode 100644 index 0000000000..96a1308d30 Binary files /dev/null and b/test/fixtures/kotlinx/kotlinx-serialization-core-jvm-1.7.3.jar differ diff --git a/test/fixtures/kotlinx/kotlinx-serialization-json-jvm-1.7.3.jar b/test/fixtures/kotlinx/kotlinx-serialization-json-jvm-1.7.3.jar new file mode 100644 index 0000000000..683271d047 Binary files /dev/null and b/test/fixtures/kotlinx/kotlinx-serialization-json-jvm-1.7.3.jar differ diff --git a/test/fixtures/kotlinx/main.kt b/test/fixtures/kotlinx/main.kt new file mode 100644 index 0000000000..fa1228a30e --- /dev/null +++ b/test/fixtures/kotlinx/main.kt @@ -0,0 +1,21 @@ +package quicktype + +import java.io.File +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json + +// explicitNulls = false makes kotlinx omit nulls when serializing, like the +// other Kotlin frameworks we test; the test harness runs this fixture with +// allowMissingNull. +val json = Json { allowStructuredMapKeys = true; explicitNulls = false } + +fun output(text: String) { + val bytes = text.toByteArray() + System.out.write(bytes, 0, bytes.size) +} + +fun main(args: Array) { + val text = File(args[0]).readText() + val top = json.decodeFromString(text) + output(json.encodeToString(top)) +} diff --git a/test/fixtures/kotlinx/run.sh b/test/fixtures/kotlinx/run.sh new file mode 100755 index 0000000000..a9238634de --- /dev/null +++ b/test/fixtures/kotlinx/run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +kotlin -cp kotlinx-serialization-core-jvm-1.7.3.jar:kotlinx-serialization-json-jvm-1.7.3.jar:main.jar quicktype.MainKt "$1" diff --git a/test/inputs/schema/prefix-items.1.fail.union.json b/test/inputs/schema/prefix-items.1.fail.union.json new file mode 100644 index 0000000000..b0603b2e33 --- /dev/null +++ b/test/inputs/schema/prefix-items.1.fail.union.json @@ -0,0 +1,4 @@ +{ + "tuple": [123, "not-a-tuple-member"], + "open": [false, 11, 13] +} diff --git a/test/inputs/schema/prefix-items.1.json b/test/inputs/schema/prefix-items.1.json new file mode 100644 index 0000000000..4dcd7ebe58 --- /dev/null +++ b/test/inputs/schema/prefix-items.1.json @@ -0,0 +1,4 @@ +{ + "tuple": [123, true], + "open": [false, 11, 13] +} diff --git a/test/inputs/schema/prefix-items.schema b/test/inputs/schema/prefix-items.schema new file mode 100644 index 0000000000..6c6d5e4f19 --- /dev/null +++ b/test/inputs/schema/prefix-items.schema @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "tuple": { + "type": "array", + "prefixItems": [{ "type": "integer" }, { "type": "boolean" }] + }, + "open": { + "type": "array", + "prefixItems": [{ "type": "boolean" }], + "items": { "type": "integer" } + } + }, + "required": ["tuple", "open"] +} diff --git a/test/languages.ts b/test/languages.ts index e268fae95f..d1a7107354 100644 --- a/test/languages.ts +++ b/test/languages.ts @@ -547,6 +547,7 @@ export const CJSONLanguage: Language = { "go-schema-pattern-properties.schema", "multi-type-enum.schema", "nested-intersection-union.schema", + "prefix-items.schema", /* Constraints (min/max and regex) are not supported (for the current implementation, can be added later, should abord parsing and return NULL) */ "minmaxlength.schema", "optional-const-ref.schema", @@ -1021,6 +1022,7 @@ I havea no idea how to encode these tests correctly. // The test driver prints the circe DecodingFailure and exits 0, so // expected-failure samples cannot be detected. "nested-intersection-union.schema", + "prefix-items.schema", "date-time-or-string.schema", "implicit-one-of.schema", "go-schema-pattern-properties.schema", @@ -1280,6 +1282,137 @@ export const KotlinJacksonLanguage: Language = { sourceFiles: ["src/language/Kotlin/index.ts"], }; +export const KotlinXLanguage: Language = { + name: "kotlin", + base: "test/fixtures/kotlinx", + compileCommand: "./build.sh", + runCommand(sample: string) { + return `./run.sh "${sample}"`; + }, + diffViaSchema: true, + skipDiffViaSchema: [ + "bug427.json", + "keywords.json", + // TODO Investigate these + "34702.json", + "76ae1.json", + ], + allowMissingNull: true, + // No "union": the kotlinx renderer emits unions as sealed classes + // without any serializer wiring, so they don't (de)serialize + // (documented TODO in KotlinXRenderer.ts). + features: ["enum", "no-defaults"], + output: "TopLevel.kt", + topLevel: "TopLevel", + skipJSON: [ + // Top-level arrays render as `typealias TopLevel = JsonArray`, + // which doesn't compile — kotlinx's JsonArray takes no type + // arguments (documented TODO in KotlinXRenderer.ts). + "bug863.json", + "github-events.json", + "optional-union.json", + "00c36.json", + "010b1.json", + "050b0.json", + "06bee.json", + "07c75.json", + "0a91a.json", + "10be4.json", + "13d8d.json", + "1a7f5.json", + "2df80.json", + "32d5c.json", + "3536b.json", + "43970.json", + "570ec.json", + "5eae5.json", + "66121.json", + "6eb00.json", + "77392.json", + "7f568.json", + "7fbfb.json", + "9847b.json", + "996bd.json", + "9a503.json", + "9eed5.json", + "a45b0.json", + "ab0d1.json", + "ad8be.json", + "b4865.json", + "c8c7e.json", + "cda6c.json", + "e2a58.json", + "e53b5.json", + "e8a0b.json", + "e8b04.json", + "f3139.json", + "f3edf.json", + "f466a.json", + // Unions render as sealed classes without serializer wiring, so + // deserialization fails at runtime (documented TODO in + // KotlinXRenderer.ts). + "combinations1.json", + "combinations2.json", + "combinations3.json", + "combinations4.json", + "kitchen-sink.json", + "nbl-stats.json", + "nst-test-suite.json", + "php-mixed-union.json", + "union-constructor-clash.json", + "unions.json", + "26c9c.json", + "29f47.json", + "33d2e.json", + "421d4.json", + "5f7fe.json", + "617e8.json", + "a0496.json", + "a3d8c.json", + "f74d5.json", + "fcca3.json", + // stringEscape renders astral-plane characters as `\u{5 hex digits}`, + // which Kotlin misparses (it only supports 4-digit `\u` escapes), so + // the @SerialName annotations don't match the JSON keys. + "blns-object.json", + "identifiers.json", + ], + skipSchema: [ + // Unions render as sealed classes without serializer wiring, so + // deserialization fails at runtime (documented TODO in + // KotlinXRenderer.ts). + "accessors.schema", + "bool-string.schema", + "class-map-union.schema", + "class-with-additional.schema", + "date-time.schema", + "description.schema", + "direct-union.schema", + "enum.schema", // enum.3.json contains an int|string union + "implicit-class-array-union.schema", + "integer-float-union.schema", + "integer-string.schema", + "minmaxlength.schema", + "multi-type-enum.schema", + "mutually-recursive.schema", + "prefix-items.schema", + "recursive-union-flattening.schema", + "tuple.schema", + "union-int-double.schema", + "union-list.schema", + // Additionally exceeds the JVM's 255-parameter limit when the + // serialization plugin generates the synthesized constructors. + "keyword-unions.schema", + // Top-level array: `typealias TopLevel = JsonArray` doesn't + // compile (documented TODO in KotlinXRenderer.ts). + "union.schema", + ], + skipMiscJSON: false, + rendererOptions: { framework: "kotlinx" }, + quickTestRendererOptions: [], + sourceFiles: ["src/language/Kotlin/index.ts"], +}; + export const DartLanguage: Language = { name: "dart", base: "test/fixtures/dart", @@ -1466,6 +1599,7 @@ export const HaskellLanguage: Language = { // The test driver encodes the Maybe result, so a failed decode prints // "null" and exits 0 — expected-failure samples cannot be detected. "nested-intersection-union.schema", + "prefix-items.schema", "direct-union.schema", ...skipsEnumValueValidation, "go-schema-pattern-properties.schema", diff --git a/test/unit/prefix-items-schema.test.ts b/test/unit/prefix-items-schema.test.ts new file mode 100644 index 0000000000..307c47b507 --- /dev/null +++ b/test/unit/prefix-items-schema.test.ts @@ -0,0 +1,89 @@ +// Regression test for issue #2811: JSON Schema 2020-12 tuple schemas that use +// `prefixItems` must be handled the same as draft-07's array-valued `items`. +// +// End-to-end coverage lives in the fixture test +// `test/inputs/schema/prefix-items.schema` (with a `.fail.union.json` sample +// that catches an `any[]` degradation). What fixtures cannot express is that +// `$ref`d tuple member types survive into the generated code, so we assert +// that directly here. + +import { + FetchingJSONSchemaStore, + InputData, + JSONSchemaInput, + quicktype, +} from "quicktype-core"; +import { describe, expect, test } from "vitest"; + +// The `$ref`'d member schemas carry distinctively named properties so we can +// assert they are present in the output rather than collapsed to `any[]`. +const defs = { + Bar: { + type: "object", + properties: { barField: { type: "integer" } }, + required: ["barField"], + }, + Baz: { + type: "object", + properties: { bazField: { type: "string" } }, + required: ["bazField"], + }, +}; + +async function generateTypeScript(schema: object): Promise { + const schemaInput = new JSONSchemaInput(new FetchingJSONSchemaStore()); + await schemaInput.addSource({ + name: "Foo", + schema: JSON.stringify(schema), + }); + const inputData = new InputData(); + inputData.addInput(schemaInput); + + const result = await quicktype({ inputData, lang: "typescript" }); + return result.lines.join("\n"); +} + +describe("JSON Schema prefixItems tuples (issue #2811)", () => { + test("a 2020-12 prefixItems tuple keeps its member types", async () => { + const output = await generateTypeScript({ + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "array", + prefixItems: [{ $ref: "#/$defs/Bar" }, { $ref: "#/$defs/Baz" }], + $defs: defs, + }); + + // Before the fix this generated a bare `any[]` with no member types. + expect(output).toContain("barField"); + expect(output).toContain("bazField"); + }); + + test("an object-form `items` next to `prefixItems` joins the union", async () => { + const output = await generateTypeScript({ + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "array", + prefixItems: [{ $ref: "#/$defs/Bar" }], + items: { $ref: "#/$defs/Baz" }, + $defs: defs, + }); + + // The rest (`items`) type must not be dropped: both the prefix + // member and the rest member appear in the element union. + expect(output).toContain("barField"); + expect(output).toContain("bazField"); + }); + + test("draft-07 array-valued items still works (no regression)", async () => { + const output = await generateTypeScript({ + $schema: "http://json-schema.org/draft-07/schema#", + type: "array", + items: [ + { $ref: "#/definitions/Bar" }, + { $ref: "#/definitions/Baz" }, + ], + definitions: defs, + }); + + expect(output).toContain("barField"); + expect(output).toContain("bazField"); + }); +}); diff --git a/test/unit/typescript-output-filename.test.ts b/test/unit/typescript-output-filename.test.ts new file mode 100644 index 0000000000..dfa789ea51 --- /dev/null +++ b/test/unit/typescript-output-filename.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from "vitest"; + +import { + InputData, + jsonInputForTargetLanguage, + quicktype, +} from "../../packages/quicktype-core/src/index.js"; + +async function generate( + lang: string, + outputFilename?: string, +): Promise { + const jsonInput = jsonInputForTargetLanguage(lang); + await jsonInput.addSource({ + name: "ExperimentCounts", + samples: ['{"experiments": 3}'], + }); + + const inputData = new InputData(); + inputData.addInput(jsonInput); + const result = await quicktype({ + inputData, + lang, + outputFilename, + }); + return result.lines.join("\n"); +} + +describe("output filename in usage comments", () => { + test("uses output filename in TypeScript usage imports", async () => { + const output = await generate( + "typescript", + "src/cli/ExperimentCounts.ts", + ); + + expect(output).toContain( + '// import { Convert, ExperimentCounts } from "./ExperimentCounts";', + ); + expect(output).not.toContain('from "./file"'); + }); + + test("uses output filename in JavaScript usage require", async () => { + const output = await generate( + "javascript", + "src/cli/ExperimentCounts.js", + ); + + expect(output).toContain( + '// const Convert = require("./ExperimentCounts");', + ); + expect(output).not.toContain('require("./file")'); + }); + + test("uses output filename in Flow usage require", async () => { + const output = await generate("flow", "ExperimentCounts.js"); + + expect(output).toContain( + '// const Convert = require("./ExperimentCounts");', + ); + expect(output).not.toContain('require("./file")'); + }); + + test('falls back to "file" when writing to stdout', async () => { + const tsOutput = await generate("typescript"); + expect(tsOutput).toContain(' } from "./file";'); + + const jsOutput = await generate("javascript"); + expect(jsOutput).toContain('// const Convert = require("./file");'); + }); +});