diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 0b1c562522..fe79e6fc67 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -53,6 +53,7 @@ jobs: - ruby - php,schema-php - scala3,schema-scala3 + - scala3-upickle,schema-scala3-upickle - elixir,schema-elixir,graphql-elixir - comment-injection-treesitter,comment-injection-typescript,comment-injection-typescript-zod,comment-injection-typescript-effect-schema @@ -124,7 +125,7 @@ jobs: if: ${{ contains(matrix.fixture, 'csharp') }} uses: actions/setup-dotnet@v3 with: - dotnet-version: 6 + dotnet-version: 8 - name: Install Rust if: ${{ contains(matrix.fixture, 'rust') }} @@ -145,16 +146,16 @@ jobs: if ! command -v stack > /dev/null 2>&1; then curl -sL "https://get.haskellstack.org/" | sh fi - - name: Install Python 3.7 + - name: Install Python 3.12 if: ${{ contains(matrix.fixture, 'python') }} uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: "3.12" - name: Install Python Dependencies if: ${{ contains(matrix.fixture, 'python') }} run: | - pip3.7 install mypy python-dateutil types-python-dateutil + pip3.12 install mypy python-dateutil types-python-dateutil - name: Install flow if: ${{ contains(matrix.fixture, 'flow') }} run: | diff --git a/package-lock.json b/package-lock.json index 3779d165a4..0436aecd16 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,9 +45,10 @@ "@types/semver": "^7.5.0", "@types/shelljs": "^0.10.0", "@types/stream-json": "^1.7.3", - "@types/urijs": "^1.19.25", + "@types/urijs": "^1.19.26", "@types/wordwrap": "^1.0.3", - "ajv": "^5.5.2", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "deep-equal": "^2.2.3", "esbuild": "^0.28.1", "promise-timeout": "^1.3.0", @@ -1444,23 +1445,6 @@ "node": ">=20.0.0" } }, - "node_modules/@secretlint/config-loader/node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/@secretlint/config-loader/node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -1479,20 +1463,6 @@ } } }, - "node_modules/@secretlint/config-loader/node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/@secretlint/config-loader/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, "node_modules/@secretlint/config-loader/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2041,12 +2011,16 @@ } }, "node_modules/@types/unicode-properties": { - "version": "1.3.0", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/unicode-properties/-/unicode-properties-1.3.2.tgz", + "integrity": "sha512-1gq48yvPn+sdJqG4tARHoQbVYyFCrV92gZdl2100vcP9n/u4nGIeLnxYyrTLWHQRhJYpc/w4SNDXUrKTjvEfRA==", "dev": true, "license": "MIT" }, "node_modules/@types/urijs": { - "version": "1.19.25", + "version": "1.19.26", + "resolved": "https://registry.npmjs.org/@types/urijs/-/urijs-1.19.26.tgz", + "integrity": "sha512-wkXrVzX5yoqLnndOwFsieJA7oKM8cNkOKJtf/3vVGSUFkWDKZvFHpIl9Pvqb/T9UsawBBFMTTD8xu7sK5MWuvg==", "license": "MIT" }, "node_modules/@types/vscode": { @@ -2969,14 +2943,38 @@ } }, "node_modules/ajv": { - "version": "5.5.2", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", "dev": true, "license": "MIT", "dependencies": { - "co": "^4.6.0", - "fast-deep-equal": "^1.0.0", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.3.0" + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } } }, "node_modules/ansi-colors": { @@ -3633,15 +3631,6 @@ "node": ">=12" } }, - "node_modules/co": { - "version": "4.6.0", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">= 1.0.0", - "node": ">= 0.12.0" - } - }, "node_modules/cockatiel": { "version": "3.1.2", "dev": true, @@ -4610,7 +4599,9 @@ "license": "MIT" }, "node_modules/fast-deep-equal": { - "version": "1.1.0", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true, "license": "MIT" }, @@ -4630,11 +4621,6 @@ "node": ">=8.6.0" } }, - "node_modules/fast-json-stable-stringify": { - "version": "2.0.0", - "dev": true, - "license": "MIT" - }, "node_modules/fast-levenshtein": { "version": "2.0.6", "dev": true, @@ -5626,7 +5612,9 @@ } }, "node_modules/json-schema-traverse": { - "version": "0.3.1", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, @@ -8598,37 +8586,6 @@ "node": ">=12.17" } }, - "node_modules/table/node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/table/node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/table/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, "node_modules/tapable": { "version": "2.2.1", "dev": true, @@ -10316,7 +10273,7 @@ "dependencies": { "@glideapps/ts-necessities": "2.2.3", "@types/readable-stream": "4.0.10", - "@types/urijs": "^1.19.25", + "@types/urijs": "^1.19.26", "browser-or-node": "^3.0.0", "collection-utils": "^1.0.1", "is-url": "^1.2.4", @@ -10333,7 +10290,7 @@ "@types/is-url": "^1.2.32", "@types/node": "~20.19.0", "@types/pluralize": "0.0.30", - "@types/unicode-properties": "^1.3.0", + "@types/unicode-properties": "^1.3.2", "@types/wordwrap": "^1.0.3", "typescript": "~7.0.2" }, diff --git a/package.json b/package.json index 09df9da7a1..9324aeb5e9 100644 --- a/package.json +++ b/package.json @@ -62,9 +62,10 @@ "@types/semver": "^7.5.0", "@types/shelljs": "^0.10.0", "@types/stream-json": "^1.7.3", - "@types/urijs": "^1.19.25", + "@types/urijs": "^1.19.26", "@types/wordwrap": "^1.0.3", - "ajv": "^5.5.2", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "deep-equal": "^2.2.3", "esbuild": "^0.28.1", "promise-timeout": "^1.3.0", diff --git a/packages/quicktype-core/package.json b/packages/quicktype-core/package.json index b57811058d..4e10b41c16 100644 --- a/packages/quicktype-core/package.json +++ b/packages/quicktype-core/package.json @@ -32,7 +32,7 @@ "dependencies": { "@glideapps/ts-necessities": "2.2.3", "@types/readable-stream": "4.0.10", - "@types/urijs": "^1.19.25", + "@types/urijs": "^1.19.26", "browser-or-node": "^3.0.0", "collection-utils": "^1.0.1", "is-url": "^1.2.4", @@ -49,7 +49,7 @@ "@types/is-url": "^1.2.32", "@types/node": "~20.19.0", "@types/pluralize": "0.0.30", - "@types/unicode-properties": "^1.3.0", + "@types/unicode-properties": "^1.3.2", "@types/wordwrap": "^1.0.3", "typescript": "~7.0.2" }, diff --git a/packages/quicktype-core/src/language/CSharp/CSharpRenderer.ts b/packages/quicktype-core/src/language/CSharp/CSharpRenderer.ts index 8dd10334da..3e331a310b 100644 --- a/packages/quicktype-core/src/language/CSharp/CSharpRenderer.ts +++ b/packages/quicktype-core/src/language/CSharp/CSharpRenderer.ts @@ -188,7 +188,7 @@ export class CSharpRenderer extends ConvenienceRenderer { ): Sourcelike { t = followTargetType(t); const csType = this.csType(t, follow, withIssues); - if (isValueType(t)) { + if (isValueType(t) || this._csOptions.version >= 8) { return [csType, "?"]; } else { return csType; diff --git a/packages/quicktype-core/src/language/CSharp/NewtonSoftCSharpRenderer.ts b/packages/quicktype-core/src/language/CSharp/NewtonSoftCSharpRenderer.ts index c69107ebd9..1993f6e135 100644 --- a/packages/quicktype-core/src/language/CSharp/NewtonSoftCSharpRenderer.ts +++ b/packages/quicktype-core/src/language/CSharp/NewtonSoftCSharpRenderer.ts @@ -192,6 +192,15 @@ export class NewtonsoftCSharpRenderer extends CSharpRenderer { return this._options.baseclass; } + protected emitDefaultFollowingComments(): void { + if (!this._needHelpers || this._options.version < 8) return; + + this.emitLine("#pragma warning restore CS8618"); + this.emitLine("#pragma warning restore CS8601"); + this.emitLine("#pragma warning restore CS8603"); + this.emitLine("#pragma warning restore CS8765"); + } + protected emitDefaultLeadingComments(): void { if (!this._needHelpers) return; @@ -225,6 +234,14 @@ export class NewtonsoftCSharpRenderer extends CSharpRenderer { ";", ); }); + + if (this._options.version >= 8) { + this.emitLine("#nullable enable"); + this.emitLine("#pragma warning disable CS8618"); + this.emitLine("#pragma warning disable CS8601"); + this.emitLine("#pragma warning disable CS8603"); + this.emitLine("#pragma warning disable CS8765"); + } } private converterForType(t: Type): Name | undefined { diff --git a/packages/quicktype-core/src/language/CSharp/SystemTextJsonCSharpRenderer.ts b/packages/quicktype-core/src/language/CSharp/SystemTextJsonCSharpRenderer.ts index 34204577b4..e64a915dcb 100644 --- a/packages/quicktype-core/src/language/CSharp/SystemTextJsonCSharpRenderer.ts +++ b/packages/quicktype-core/src/language/CSharp/SystemTextJsonCSharpRenderer.ts @@ -275,6 +275,13 @@ export class SystemTextJsonCSharpRenderer extends CSharpRenderer { const isNullable = followTargetType(property.type).isNullable; const isOptional = property.isOptional; + // [JsonRequired] makes deserialization fail if the property is + // missing from the JSON. It requires System.Text.Json 7.0 or + // later (.NET 7+). + if (this._options.checkRequired && !isOptional) { + attributes.push(["[JsonRequired]"]); + } + if (isOptional && !isNullable) { attributes.push([ "[", diff --git a/packages/quicktype-core/src/language/CSharp/language.ts b/packages/quicktype-core/src/language/CSharp/language.ts index 0b0c7235f5..b7447d7329 100644 --- a/packages/quicktype-core/src/language/CSharp/language.ts +++ b/packages/quicktype-core/src/language/CSharp/language.ts @@ -69,6 +69,7 @@ export const cSharpOptions = { { "5": 5, "6": 6, + "8": 8, } as const, "6", "secondary", diff --git a/packages/quicktype-core/src/language/JavaScript/JavaScriptRenderer.ts b/packages/quicktype-core/src/language/JavaScript/JavaScriptRenderer.ts index 0adcaefd30..c14a44efdb 100644 --- a/packages/quicktype-core/src/language/JavaScript/JavaScriptRenderer.ts +++ b/packages/quicktype-core/src/language/JavaScript/JavaScriptRenderer.ts @@ -239,6 +239,16 @@ export class JavaScriptRenderer extends ConvenienceRenderer { }; } + /** The expression a deserializer returns when runtime typechecks + * are disabled. Subclasses can wrap it in a cast to the target + * type if `parsedJson`'s type isn't assignable to it. */ + protected uncheckedParsedJson( + _t: Type, + parsedJson: Sourcelike, + ): Sourcelike { + return parsedJson; + } + protected emitConvertModuleBody(): void { const converter = (t: Type, name: Name): void => { const typeMap = this.typeMapTypeFor(t); @@ -251,7 +261,11 @@ export class JavaScriptRenderer extends ConvenienceRenderer { ? "JSON.parse(json)" : "json"; if (!this._jsOptions.runtimeTypecheck) { - this.emitLine("return ", parsedJson, ";"); + this.emitLine( + "return ", + this.uncheckedParsedJson(t, parsedJson), + ";", + ); } else { this.emitLine( "return cast(", diff --git a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts index 59271b599c..20d5fc5aa8 100644 --- a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts @@ -370,7 +370,15 @@ export class JSONPythonRenderer extends PythonRenderer { ", ", this.typingDecl("x", "Any"), ")", - this.typeHint(" -> ", this.withTyping("List"), "[", tvar, "]"), + this.typeHint( + " -> ", + this.pyOptions.features.builtinGenerics + ? "list" + : this.withTyping("List"), + "[", + tvar, + "]", + ), ":", ], () => { @@ -422,7 +430,9 @@ export class JSONPythonRenderer extends PythonRenderer { ")", this.typeHint( " -> ", - this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics + ? "dict" + : this.withTyping("Dict"), "[str, ", tvar, "]", @@ -611,7 +621,8 @@ export class JSONPythonRenderer extends PythonRenderer { (_integerType) => "int", (_doubleType) => "float", (_stringType) => "str", - (_arrayType) => "List", + (_arrayType) => + this.pyOptions.features.builtinGenerics ? "list" : "List", (classType) => this.nameForNamedType(classType), (_mapType) => "dict", (enumType) => this.nameForNamedType(enumType), diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index 09072ca05f..e0d60263ab 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -1,6 +1,7 @@ import { arrayIntercalate, iterableFirst, + iterableSome, mapSortBy, mapUpdateInto, setUnionInto, @@ -19,17 +20,15 @@ import { defined, panic } from "../../support/Support.js"; import type { TargetLanguage } from "../../TargetLanguage.js"; import { followTargetType } from "../../Transformers.js"; import { + ArrayType, type ClassProperty, ClassType, EnumType, + MapType, type Type, UnionType, } from "../../Type/index.js"; -import { - matchType, - nullableFromUnion, - removeNullFromUnion, -} from "../../Type/TypeUtils.js"; +import { matchType, removeNullFromUnion } from "../../Type/TypeUtils.js"; import { forbiddenPropertyNames, forbiddenTypeNames } from "./constants.js"; import type { pythonOptions } from "./language.js"; @@ -137,56 +136,160 @@ export class PythonRenderer extends ConvenienceRenderer { return this.withImport("typing", name); } - protected namedType(t: Type): Sourcelike { + protected namedType(t: Type, suppressQuotes = false): Sourcelike { const name = this.nameForNamedType(t); - if (this.declaredTypes.has(t)) return name; + if (suppressQuotes || this.declaredTypes.has(t)) return name; return ["'", name, "'"]; } - protected pythonType(t: Type, _isRootTypeDef = false): Sourcelike { + // Would rendering `t` as a type annotation right now require a forward + // reference, i.e. does it mention a named type that hasn't been declared + // yet? + private typeContainsForwardRef(t: Type): boolean { + const actualType = followTargetType(t); + if (actualType instanceof ClassType || actualType instanceof EnumType) { + return !this.declaredTypes.has(actualType); + } + + if (actualType instanceof ArrayType) { + return this.typeContainsForwardRef(actualType.items); + } + + if (actualType instanceof MapType) { + return this.typeContainsForwardRef(actualType.values); + } + + if (actualType instanceof UnionType) { + return iterableSome(actualType.members, (m) => + this.typeContainsForwardRef(m), + ); + } + + return false; + } + + // Renders a union with PEP 604 syntax: `A | B | None`. A quoted forward + // reference is not allowed as an operand of `|` (`'A' | None` is a runtime + // `TypeError`), so if any member needs a forward reference we quote the + // whole union expression instead, suppressing the quotes on the individual + // names. A `" = None"` default, if required, must go outside the closing + // quote: `foo: 'Foo | None' = None`. + private pep604UnionType( + unionType: UnionType, + isRootTypeDef: boolean, + suppressQuotes: boolean, + ): Sourcelike { + const [hasNull, nonNulls] = removeNullFromUnion(unionType); + const needsQuotes = + !suppressQuotes && + iterableSome(nonNulls, (m) => this.typeContainsForwardRef(m)); + const quote = needsQuotes ? "'" : ""; + const memberTypes = Array.from(nonNulls).map((m) => + this.pythonType(m, false, true), + ); + + const union: Sourcelike[] = [ + quote, + arrayIntercalate(" | ", memberTypes), + ]; + if (hasNull !== null) { + union.push(" | None"); + } + + union.push(quote); + + if (hasNull !== null) { + union.push(...this.noneDefault(isRootTypeDef)); + } + + return union; + } + + // A `" = None"` default for a class property whose value can be `None`. + // Only emitted for root level type defs, otherwise we may get type defs + // like `List[Optional[int] = None]`, which are invalid. Every property + // that gets a default must sort after all properties that don't — see + // `sortClassProperties`. + private noneDefault(isRootTypeDef: boolean): string[] { + if ( + isRootTypeDef && + !this.getAlphabetizeProperties() && + (this.pyOptions.features.dataClasses || + this.pyOptions.pydanticBaseModel) + ) { + return [" = None"]; + } + + return []; + } + + // Does `pythonType(p.type, true)` end in a `" = None"` default? This + // must mirror the `noneDefault` calls in `pythonType` exactly: nullable + // unions, plus `Any` and `None` typed properties — an optional `Any` + // stays `Any` (`null` is absorbed by it), and an optional `null` + // collapses to just `null`, so those also default to `None`. + private classPropertyHasNoneDefault(p: ClassProperty): boolean { + const actualType = followTargetType(p.type); + if (actualType instanceof UnionType) { + const [hasNull] = removeNullFromUnion(actualType); + return hasNull !== null; + } + + return actualType.kind === "any" || actualType.kind === "null"; + } + + protected pythonType( + t: Type, + _isRootTypeDef = false, + suppressQuotes = false, + ): Sourcelike { const actualType = followTargetType(t); return matchType( actualType, - (_anyType) => this.withTyping("Any"), - (_nullType) => "None", + (_anyType) => [ + this.withTyping("Any"), + ...this.noneDefault(_isRootTypeDef), + ], + (_nullType) => ["None", ...this.noneDefault(_isRootTypeDef)], (_boolType) => "bool", (_integerType) => "int", (_doubletype) => "float", (_stringType) => "str", (arrayType) => [ - this.withTyping("List"), + this.pyOptions.features.builtinGenerics + ? "list" + : this.withTyping("List"), "[", - this.pythonType(arrayType.items), + this.pythonType(arrayType.items, false, suppressQuotes), "]", ], - (classType) => this.namedType(classType), + (classType) => this.namedType(classType, suppressQuotes), (mapType) => [ - this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics + ? "dict" + : this.withTyping("Dict"), "[str, ", - this.pythonType(mapType.values), + this.pythonType(mapType.values, false, suppressQuotes), "]", ], - (enumType) => this.namedType(enumType), + (enumType) => this.namedType(enumType, suppressQuotes), (unionType) => { + if (this.pyOptions.features.unionOperators) { + return this.pep604UnionType( + unionType, + _isRootTypeDef, + suppressQuotes, + ); + } + const [hasNull, nonNulls] = removeNullFromUnion(unionType); const memberTypes = Array.from(nonNulls).map((m) => - this.pythonType(m), + this.pythonType(m, false, suppressQuotes), ); if (hasNull !== null) { - const rest: string[] = []; - if ( - !this.getAlphabetizeProperties() && - (this.pyOptions.features.dataClasses || - this.pyOptions.pydanticBaseModel) && - _isRootTypeDef - ) { - // Only push "= None" if this is a root level type def - // otherwise we may get type defs like List[Optional[int] = None] - // which are invalid - rest.push(" = None"); - } + const rest = this.noneDefault(_isRootTypeDef); if (nonNulls.size > 1) { this.withImport("typing", "Union"); @@ -321,13 +424,11 @@ export class PythonRenderer extends ConvenienceRenderer { this.pyOptions.features.dataClasses || this.pyOptions.pydanticBaseModel ) { - return mapSortBy(properties, (p: ClassProperty) => { - return (p.type instanceof UnionType && - nullableFromUnion(p.type) !== null) || - p.isOptional - ? 1 - : 0; - }); + // Properties that get a `" = None"` default must come after all + // properties that don't, or the generated dataclass is invalid. + return mapSortBy(properties, (p: ClassProperty) => + this.classPropertyHasNoneDefault(p) ? 1 : 0, + ); } return super.sortClassProperties(properties, propertyNames); diff --git a/packages/quicktype-core/src/language/Python/constants.ts b/packages/quicktype-core/src/language/Python/constants.ts index d32c109235..1c96606977 100644 --- a/packages/quicktype-core/src/language/Python/constants.ts +++ b/packages/quicktype-core/src/language/Python/constants.ts @@ -5,7 +5,9 @@ export const forbiddenTypeNames = [ "None", "Enum", "List", + "list", "Dict", + "dict", "Optional", "Union", "Iterable", diff --git a/packages/quicktype-core/src/language/Python/language.ts b/packages/quicktype-core/src/language/Python/language.ts index 101c5115b8..263026b7e5 100644 --- a/packages/quicktype-core/src/language/Python/language.ts +++ b/packages/quicktype-core/src/language/Python/language.ts @@ -21,8 +21,12 @@ import { JSONPythonRenderer } from "./JSONPythonRenderer.js"; import { PythonRenderer } from "./PythonRenderer.js"; export interface PythonFeatures { + /** PEP 585 builtin generics (`list[str]`, `dict[str, int]`), Python 3.9+ */ + builtinGenerics: boolean; dataClasses: boolean; typeHints: boolean; + /** PEP 604 union operators (`str | None`), Python 3.10+ */ + unionOperators: boolean; } export const pythonOptions = { @@ -30,11 +34,38 @@ export const pythonOptions = { "python-version", "Python version", { - "3.5": { typeHints: false, dataClasses: false }, - "3.6": { typeHints: true, dataClasses: false }, - "3.7": { typeHints: true, dataClasses: true }, - }, - "3.6", + "3.5": { + typeHints: false, + dataClasses: false, + builtinGenerics: false, + unionOperators: false, + }, + "3.6": { + typeHints: true, + dataClasses: false, + builtinGenerics: false, + unionOperators: false, + }, + "3.7": { + typeHints: true, + dataClasses: true, + builtinGenerics: false, + unionOperators: false, + }, + "3.9": { + typeHints: true, + dataClasses: true, + builtinGenerics: true, + unionOperators: false, + }, + "3.10": { + typeHints: true, + dataClasses: true, + builtinGenerics: true, + unionOperators: true, + }, + } satisfies Record, + "3.10", ), justTypes: new BooleanOption("just-types", "Classes only", false), nicePropertyNames: new BooleanOption( diff --git a/packages/quicktype-core/src/language/Scala3/CirceRenderer.ts b/packages/quicktype-core/src/language/Scala3/CirceRenderer.ts index 5f513c0c5f..fc4a23dd39 100644 --- a/packages/quicktype-core/src/language/Scala3/CirceRenderer.ts +++ b/packages/quicktype-core/src/language/Scala3/CirceRenderer.ts @@ -14,8 +14,10 @@ import type { UnionType, } from "../../Type/index.js"; +import { stringEscape } from "../../support/Strings.js"; + import { Scala3Renderer } from "./Scala3Renderer.js"; -import { wrapOption } from "./utils.js"; +import { unionMemberSortOrder, wrapOption } from "./utils.js"; export class CirceRenderer extends Scala3Renderer { private readonly seenUnionTypes: string[] = []; @@ -55,7 +57,13 @@ export class CirceRenderer extends Scala3Renderer { paramName, ")", ], - (_) => ["Encoder.encodeString(", paramName, ")"], + (enumType) => [ + "summon[Encoder[", + this.scalaType(enumType), + "]].apply(", + paramName, + ")", + ], (unionType) => { const nullable = nullableFromUnion(unionType); if (nullable !== null) { @@ -103,44 +111,70 @@ export class CirceRenderer extends Scala3Renderer { protected emitEnumDefinition(e: EnumType, enumName: Name): void { this.emitDescription(this.descriptionForType(e)); + this.ensureBlankLine(); + // Enum cases are styled Scala identifiers; the codecs below map + // them back to the original JSON strings, which can be anything + // (keywords, `"_"`, `""`, …). + this.emitLine(["enum ", enumName, " : "]); + this.indent(() => { + this.forEachEnumCase(e, "none", (name) => { + this.emitLine("case ", name); + }); + }); this.ensureBlankLine(); - this.emitItem(["type ", enumName, " = "]); - let count = e.cases.size; - this.forEachEnumCase(e, "none", (_, jsonName) => { - // if (!(jsonName == "")) { - /* const backticks = - shouldAddBacktick(jsonName) || - jsonName.includes(" ") || - !isNaN(parseInt(jsonName.charAt(0))) - if (backticks) {this.emitItem("`")} else */ - this.emitItem(['"', jsonName, '"']); - // if (backticks) {this.emitItem("`")} - if (--count > 0) this.emitItem([" | "]); - // } else { - // --count - // } + + this.emitLine([ + "given Decoder[", + enumName, + "] = Decoder.decodeString.emap {", + ]); + this.indent(() => { + // `scala.` in case a generated type is named Right/Left. + this.forEachEnumCase(e, "none", (name, jsonName) => { + this.emitLine([ + `case "${stringEscape(jsonName)}" => scala.Right(`, + enumName, + ".", + name, + ")", + ]); + }); + this.emitLine([ + 'case other => scala.Left("invalid ', + enumName, + ': " + other)', + ]); }); + this.emitLine("}"); + this.emitLine([ + "given Encoder[", + enumName, + "] = Encoder.encodeString.contramap {", + ]); + this.indent(() => { + this.forEachEnumCase(e, "none", (name, jsonName) => { + this.emitLine([ + "case ", + enumName, + ".", + name, + ` => "${stringEscape(jsonName)}"`, + ]); + }); + }); + this.emitLine("}"); this.ensureBlankLine(); } protected emitHeader(): void { super.emitHeader(); - this.emitLine("import scala.util.Try"); this.emitLine("import io.circe.syntax._"); this.emitLine("import io.circe._"); this.emitLine("import cats.syntax.functor._"); this.ensureBlankLine(); - this.emitLine("// For serialising string unions"); - this.emitLine( - "given [A <: Singleton](using A <:< String): Decoder[A] = Decoder.decodeString.emapTry(x => Try(x.asInstanceOf[A])) ", - ); - this.emitLine( - "given [A <: Singleton](using ev: A <:< String): Encoder[A] = Encoder.encodeString.contramap(ev) ", - ); - this.ensureBlankLine(); this.emitLine( "// If a union has a null in, then we'll need this too... ", ); @@ -153,9 +187,9 @@ export class CirceRenderer extends Scala3Renderer { this.emitLine([ "given (using ev : ", elementType, - "): Encoder[Map[String,", + "): Encoder[Seq[", elementType, - "]] = Encoder.encodeMap[String, ", + "]] = Encoder.encodeSeq[", elementType, "]", ]); @@ -177,15 +211,9 @@ export class CirceRenderer extends Scala3Renderer { } protected emitUnionDefinition(u: UnionType, unionName: Name): void { - function sortBy(t: Type): string { - const kind = t.kind; - if (kind === "class") return kind; - return `_${kind}`; - } - this.emitDescription(this.descriptionForType(u)); - const [maybeNull, nonNulls] = removeNullFromUnion(u, sortBy); + const [maybeNull, nonNulls] = removeNullFromUnion(u, false); const theTypes: Sourcelike[] = []; this.forEachUnionMember(u, nonNulls, "none", null, (_, t) => { theTypes.push(this.scalaType(t)); @@ -205,10 +233,20 @@ export class CirceRenderer extends Scala3Renderer { this.ensureBlankLine(); if (!this.seenUnionTypes.some((y) => y === thisUnionType)) { this.seenUnionTypes.push(thisUnionType); + // The decoders are tried in order, most discriminating + // first (see `unionMemberSortOrder`): circe's numeric + // decoders accept strings like "5", so the string decoder + // has to get a shot before them. const sourceLikeTypes: Array<[Sourcelike, Type]> = []; - this.forEachUnionMember(u, nonNulls, "none", null, (_, t) => { - sourceLikeTypes.push([this.scalaType(t), t]); - }); + this.forEachUnionMember( + u, + nonNulls, + "none", + unionMemberSortOrder, + (_, t) => { + sourceLikeTypes.push([this.scalaType(t), t]); + }, + ); if (maybeNull !== null) { sourceLikeTypes.push([ this.nameForUnionMember(u, maybeNull), diff --git a/packages/quicktype-core/src/language/Scala3/Scala3Renderer.ts b/packages/quicktype-core/src/language/Scala3/Scala3Renderer.ts index 34234f728e..09b6e59fcf 100644 --- a/packages/quicktype-core/src/language/Scala3/Scala3Renderer.ts +++ b/packages/quicktype-core/src/language/Scala3/Scala3Renderer.ts @@ -30,9 +30,10 @@ import { import { keywords } from "./constants.js"; import type { scala3Options } from "./language.js"; import { + backtickedName, + enumCaseNameStyle, lowerNamingFunction, scalaNameStyle, - shouldAddBacktick, upperNamingFunction, wrapOption, } from "./utils.js"; @@ -61,7 +62,7 @@ export class Scala3Renderer extends ConvenienceRenderer { _: EnumType, _enumName: Name, ): ForbiddenWordsInfo { - return { names: [], includeGlobalForbidden: true }; + return { names: ["_"], includeGlobalForbidden: true }; } protected forbiddenForUnionMembers( @@ -91,7 +92,7 @@ export class Scala3Renderer extends ConvenienceRenderer { } protected makeEnumCaseNamer(): Namer { - return funPrefixNamer("upper", (s) => s.replace(" ", "")); // TODO - add backticks where appropriate + return funPrefixNamer("upper", enumCaseNameStyle); } protected emitDescriptionBlock(lines: Sourcelike[]): void { @@ -262,14 +263,9 @@ export class Scala3Renderer extends ConvenienceRenderer { emit(); } - const nameNeedsBackticks = - jsonName.endsWith("_") || shouldAddBacktick(jsonName); - const nameWithBackticks = nameNeedsBackticks - ? `\`${jsonName}\`` - : jsonName; this.emitLine( "val ", - nameWithBackticks, + backtickedName(jsonName), " : ", scalaType(p), p.isOptional @@ -301,32 +297,8 @@ export class Scala3Renderer extends ConvenienceRenderer { this.emitBlock( ["enum ", enumName, " : "], () => { - let count = e.cases.size; - if (count > 0) { - this.emitItem("\t case "); - } - - this.forEachEnumCase(e, "none", (name, jsonName) => { - if (!(jsonName === "")) { - const backticks = - shouldAddBacktick(jsonName) || - jsonName.includes(" ") || - !Number.isNaN( - Number.parseInt(jsonName.charAt(0), 10), - ); - if (backticks) { - this.emitItem("`"); - } - - this.emitItemOnce([name]); - if (backticks) { - this.emitItem("`"); - } - - if (--count > 0) this.emitItem([","]); - } else { - --count; - } + this.forEachEnumCase(e, "none", (name) => { + this.emitLine("case ", name); }); }, "none", @@ -361,12 +333,14 @@ export class Scala3Renderer extends ConvenienceRenderer { protected emitSourceStructure(): void { this.emitHeader(); - // Top-level arrays, maps + // Top-level arrays, maps, and primitives this.forEachTopLevel("leading", (t, name) => { if (t instanceof ArrayType) { this.emitTopLevelArray(t, name); } else if (t instanceof MapType) { this.emitTopLevelMap(t, name); + } else if (t.isPrimitive()) { + this.emitLine(["type ", name, " = ", this.scalaType(t)]); } }); diff --git a/packages/quicktype-core/src/language/Scala3/UpickleRenderer.ts b/packages/quicktype-core/src/language/Scala3/UpickleRenderer.ts index d985ed1694..bc274f3c4d 100644 --- a/packages/quicktype-core/src/language/Scala3/UpickleRenderer.ts +++ b/packages/quicktype-core/src/language/Scala3/UpickleRenderer.ts @@ -1,14 +1,263 @@ +import type { Name } from "../../Naming.js"; +import type { Sourcelike } from "../../Source.js"; +import { removeNullFromUnion } from "../../Type/TypeUtils.js"; +import type { ClassType, EnumType, Type, UnionType } from "../../Type/index.js"; +import { stringEscape } from "../../support/Strings.js"; + import { Scala3Renderer } from "./Scala3Renderer.js"; +import { unionMemberSortOrder, wrapOption } from "./utils.js"; export class UpickleRenderer extends Scala3Renderer { + private readonly seenUnionTypes: string[] = []; + protected emitClassDefinitionMethods(): void { - this.emitLine(") derives ReadWriter "); + this.emitLine(") derives OptionPickler.ReadWriter"); + } + + protected anySourceType(optional: boolean): Sourcelike { + return [wrapOption("ujson.Value", optional)]; } protected emitHeader(): void { super.emitHeader(); + this.emitMultiline(`// Custom pickler so that missing keys and JSON nulls both read as None, +// and None is left out when writing (upickle's default for Option is a +// JSON array). +object OptionPickler extends upickle.AttributeTagged: + import upickle.default.Writer + import upickle.default.Reader + override implicit def OptionWriter[T: Writer]: Writer[Option[T]] = + implicitly[Writer[T]].comap[Option[T]] { + case None => null.asInstanceOf[T] + case Some(x) => x + } + + override implicit def OptionReader[T: Reader]: Reader[Option[T]] = { + new Reader.Delegate[Any, Option[T]](implicitly[Reader[T]].map(Some(_))){ + override def visitNull(index: Int) = None + } + } +end OptionPickler + +// If a union has a null in, then we'll need this too... +type NullValue = None.type +given OptionPickler.ReadWriter[NullValue] = OptionPickler.readwriter[ujson.Value].bimap[NullValue]( + _ => ujson.Null, + json => if json.isNull then None else throw new upickle.core.Abort("not null") +) + +object JsonExt: + val valueReader = OptionPickler.readwriter[ujson.Value] + + // upickle's built-in primitive readers are lenient -- the numeric and + // boolean readers accept strings, and the string reader accepts + // numbers and booleans -- so untagged unions need strict readers to + // pick the right member. + val strictString: OptionPickler.Reader[String] = valueReader.map { + case ujson.Str(s) => s + case json => throw new upickle.core.Abort("expected string, got " + json) + } + val strictLong: OptionPickler.Reader[Long] = valueReader.map { + case ujson.Num(n) if n.isWhole => n.toLong + case json => throw new upickle.core.Abort("expected integer, got " + json) + } + val strictDouble: OptionPickler.Reader[Double] = valueReader.map { + case ujson.Num(n) => n + case json => throw new upickle.core.Abort("expected number, got " + json) + } + val strictBoolean: OptionPickler.Reader[Boolean] = valueReader.map { + case ujson.Bool(b) => b + case json => throw new upickle.core.Abort("expected boolean, got " + json) + } + + def badMerge[T](r1: => OptionPickler.Reader[?], rest: OptionPickler.Reader[?]*): OptionPickler.Reader[T] = valueReader.map { json => + var t: T | Null = null + val stack = Vector.newBuilder[Throwable] + (r1 +: rest).foreach { reader => + if t == null then + try + t = OptionPickler.read[T](json, trace = true)(using reader.asInstanceOf[OptionPickler.Reader[T]]) + catch + case exc => stack += exc + } + if t != null then t.nn else throw new Exception(json.toString(), stack.result().headOption.getOrElse(null)) + } +end JsonExt +`); + this.ensureBlankLine(); + } + + protected override emitEmptyClassDefinition( + c: ClassType, + className: Name, + ): void { + super.emitEmptyClassDefinition(c, className); + this.emitItem(" derives OptionPickler.ReadWriter"); + this.ensureBlankLine(); + } + + protected override emitUnionDefinition( + u: UnionType, + unionName: Name, + ): void { + this.emitDescription(this.descriptionForType(u)); + + const [maybeNull, nonNulls] = removeNullFromUnion(u, false); + const theTypes: Sourcelike[] = []; + this.forEachUnionMember(u, nonNulls, "none", null, (_, t) => { + theTypes.push(this.scalaType(t)); + }); + if (maybeNull !== null) { + theTypes.push(this.nameForUnionMember(u, maybeNull)); + } + + this.emitItem(["type ", unionName, " = "]); + theTypes.forEach((t, i) => { + this.emitItem(i === 0 ? t : [" | ", t]); + }); + const thisUnionType = theTypes + .map((x) => this.sourcelikeToString(x)) + .join(" | "); + + this.ensureBlankLine(); + if (!this.seenUnionTypes.some((y) => y === thisUnionType)) { + this.seenUnionTypes.push(thisUnionType); + const sourceLikeTypes: Array<[Sourcelike, Type]> = []; + this.forEachUnionMember( + u, + nonNulls, + "none", + unionMemberSortOrder, + (_, t) => { + sourceLikeTypes.push([this.scalaType(t), t]); + }, + ); + if (maybeNull !== null) { + sourceLikeTypes.push([ + this.nameForUnionMember(u, maybeNull), + maybeNull, + ]); + } + + this.ensureBlankLine(); + this.emitLine([ + "given unionReader", + unionName, + ": OptionPickler.Reader[", + unionName, + "] = JsonExt.badMerge[", + unionName, + "](", + ]); + this.indent(() => { + sourceLikeTypes.forEach(([srcType, t]) => { + // Use the strict readers for primitive members -- + // upickle's built-in ones accept too much (see the + // comment in the emitted JsonExt). + const strictReaders: Partial> = { + Boolean: "JsonExt.strictBoolean", + Double: "JsonExt.strictDouble", + Long: "JsonExt.strictLong", + String: "JsonExt.strictString", + }; + const strict = t.isPrimitive() + ? strictReaders[this.sourcelikeToString(srcType)] + : undefined; + if (strict === undefined) { + this.emitLine([ + "summon[OptionPickler.Reader[", + srcType, + "]],", + ]); + } else { + this.emitLine([strict, ","]); + } + }); + this.emitLine(")"); + }); + this.ensureBlankLine(); + this.emitLine([ + "given unionWriter", + unionName, + ": OptionPickler.Writer[", + unionName, + "] = OptionPickler.writer[ujson.Value].comap[", + unionName, + "]{ _v =>", + ]); + this.indent(() => { + this.emitLine("(_v: @unchecked) match "); + this.indent(() => { + sourceLikeTypes.forEach((t) => { + this.emitLine([ + "case v: ", + t[0], + " => OptionPickler.writeJs[", + t[0], + "](v)", + ]); + }); + }); + }); + this.emitLine("}"); + } + } + + protected emitEnumDefinition(e: EnumType, enumName: Name): void { + this.emitDescription(this.descriptionForType(e)); + this.ensureBlankLine(); + + // Enum cases are styled Scala identifiers; the ReadWriter below + // maps them back to the original JSON strings, which can be + // anything (keywords, `"_"`, `""`, …). + this.emitLine(["enum ", enumName, " : "]); + this.indent(() => { + this.forEachEnumCase(e, "none", (name) => { + this.emitLine("case ", name); + }); + }); + this.ensureBlankLine(); - this.emitLine("import upickle.default.*"); + this.emitLine([ + "given OptionPickler.ReadWriter[", + enumName, + "] = OptionPickler.readwriter[String].bimap[", + enumName, + "](", + ]); + this.indent(() => { + this.emitLine("{"); + this.indent(() => { + this.forEachEnumCase(e, "none", (name, jsonName) => { + this.emitLine([ + "case ", + enumName, + ".", + name, + ` => "${stringEscape(jsonName)}"`, + ]); + }); + }); + this.emitLine("},"); + this.emitLine("{"); + this.indent(() => { + this.forEachEnumCase(e, "none", (name, jsonName) => { + this.emitLine([ + `case "${stringEscape(jsonName)}" => `, + enumName, + ".", + name, + ]); + }); + this.emitLine([ + 'case other => throw new upickle.core.Abort("invalid ', + enumName, + ': " + other)', + ]); + }); + this.emitLine("}"); + }); + this.emitLine(")"); this.ensureBlankLine(); } } diff --git a/packages/quicktype-core/src/language/Scala3/constants.ts b/packages/quicktype-core/src/language/Scala3/constants.ts index 918172d7fe..5cdbd2b545 100644 --- a/packages/quicktype-core/src/language/Scala3/constants.ts +++ b/packages/quicktype-core/src/language/Scala3/constants.ts @@ -1,5 +1,6 @@ // Use backticks for param names with symbols export const invalidSymbols = [ + "?", ":", "-", "+", diff --git a/packages/quicktype-core/src/language/Scala3/utils.ts b/packages/quicktype-core/src/language/Scala3/utils.ts index 3c7258dc44..638855d9b7 100644 --- a/packages/quicktype-core/src/language/Scala3/utils.ts +++ b/packages/quicktype-core/src/language/Scala3/utils.ts @@ -1,4 +1,5 @@ -import { funPrefixNamer } from "../../Naming.js"; +import { type Name, funPrefixNamer } from "../../Naming.js"; +import type { Type } from "../../Type/index.js"; import { allLowerWordStyle, allUpperWordStyle, @@ -26,6 +27,15 @@ export const shouldAddBacktick = (paramName: string): boolean => { ); }; +/** + * Wrap a name in backticks if it isn't usable as a plain Scala identifier. + */ +export const backtickedName = (name: string): string => { + return name.endsWith("_") || name.includes(" ") || shouldAddBacktick(name) + ? `\`${name}\`` + : name; +}; + export const wrapOption = (s: string, optional: boolean): string => { if (optional) { return `Option[${s}]`; @@ -34,6 +44,29 @@ export const wrapOption = (s: string, optional: boolean): string => { } }; +/** + * Sort order for union members when emitting the decoders of an untagged + * union, which try each member in turn: both circe's and upickle's + * primitive number/boolean decoders are lenient (they accept strings like + * "5"), and a case class whose fields are all defaulted can spuriously + * read non-object JSON with upickle, so try the most discriminating + * decoders first -- enums (which accept only their known strings), then + * strings, then the other primitives, classes last. + */ +export const unionMemberSortOrder = (_: Name, t: Type): string => { + const priority: Partial> = { + enum: "0", + string: "2", + bool: "3", + integer: "4", + double: "5", + array: "6", + map: "7", + class: "9", + }; + return `${priority[t.kind] ?? "8"}${t.kind}`; +}; + function isPartCharacter(codePoint: number): boolean { return isLetterOrUnderscore(codePoint) || isNumeric(codePoint); } @@ -58,16 +91,19 @@ export function scalaNameStyle(isUpper: boolean, original: string): string { ); } -/* function unicodeEscape(codePoint: number): string { - return "\\u" + intToHex(codePoint, 4); -} */ - -// const _stringEscape = utf32ConcatMap(escapeNonPrintableMapper(isPrintable, unicodeEscape)); - -/* function stringEscape(s: string): string { - // "$this" is a template string in Kotlin so we have to escape $ - return _stringEscape(s).replace(/\$/g, "\\$"); -} */ +/** + * Style an enum case as a legal Scala identifier. The JSON string an enum + * case comes from can be anything (a keyword, `"_"`, `""`, …), so the + * renderers emit styled case names and map them back to the original JSON + * strings in their codecs. + */ +export function enumCaseNameStyle(original: string): string { + const styled = scalaNameStyle(true, original); + // `scalaNameStyle` can produce the empty string, for example for `""` + // or `"_"` (which is not a legal identifier even in backticks). The + // namer disambiguates if an enum has several such cases. + return styled === "" ? "Empty" : styled; +} export const upperNamingFunction = funPrefixNamer("upper", (s) => scalaNameStyle(true, s), diff --git a/packages/quicktype-core/src/language/TypeScriptFlow/FlowRenderer.ts b/packages/quicktype-core/src/language/TypeScriptFlow/FlowRenderer.ts index a0a3bfaee2..9e48bfb147 100644 --- a/packages/quicktype-core/src/language/TypeScriptFlow/FlowRenderer.ts +++ b/packages/quicktype-core/src/language/TypeScriptFlow/FlowRenderer.ts @@ -1,17 +1,42 @@ import type { Name } from "../../Naming.js"; +import type { Sourcelike } from "../../Source.js"; import { utf16StringEscape } from "../../support/Strings.js"; import { defined } from "../../support/Support.js"; -import type { ClassType, EnumType } from "../../Type/index.js"; +import type { ClassType, EnumType, Type } from "../../Type/index.js"; import type { JavaScriptTypeAnnotations } from "../JavaScript/index.js"; import { TypeScriptFlowBaseRenderer } from "./TypeScriptFlowBaseRenderer.js"; import { tsFlowTypeAnnotations } from "./utils.js"; export class FlowRenderer extends TypeScriptFlowBaseRenderer { + protected anyType(): string { + return this._tsFlowOptions.preferUnknown ? "mixed" : "any"; + } + protected forbiddenNamesForGlobalNamespace(): string[] { return ["Class", "Date", "Object", "String", "Array", "JSON", "Error"]; } + protected uncheckedParsedJson(t: Type, parsedJson: Sourcelike): Sourcelike { + // With `raw-type any` and `prefer-unknown` the deserializer's + // parameter is `mixed`, which can't be returned as the target + // type without a cast. + if ( + this._tsFlowOptions.rawType !== "json" && + this._tsFlowOptions.preferUnknown + ) { + return [ + "((", + parsedJson, + ": any): ", + this.sourceFor(t).source, + ")", + ]; + } + + return parsedJson; + } + protected get typeAnnotations(): JavaScriptTypeAnnotations { return { never: "", ...tsFlowTypeAnnotations }; } diff --git a/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptFlowBaseRenderer.ts b/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptFlowBaseRenderer.ts index d20f56c19c..d833317d86 100644 --- a/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptFlowBaseRenderer.ts +++ b/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptFlowBaseRenderer.ts @@ -64,7 +64,7 @@ export abstract class TypeScriptFlowBaseRenderer extends JavaScriptRenderer { return matchType( t, - (_anyType) => singleWord("any"), + (_anyType) => singleWord(this.anyType()), (_nullType) => singleWord("null"), (_boolType) => singleWord("boolean"), (_integerType) => singleWord("number"), @@ -113,6 +113,12 @@ export abstract class TypeScriptFlowBaseRenderer extends JavaScriptRenderer { ); } + /** The type emitted for `any`-typed values in type declarations + * and converter signatures: the language's type-safe top type + * (`unknown` for TypeScript, `mixed` for Flow) with the + * `prefer-unknown` option, plain `any` without it. */ + protected abstract anyType(): string; + protected abstract emitEnum(e: EnumType, enumName: Name): void; protected abstract emitClassBlock(c: ClassType, className: Name): void; @@ -198,7 +204,7 @@ export abstract class TypeScriptFlowBaseRenderer extends JavaScriptRenderer { protected deserializerFunctionLine(t: Type, name: Name): Sourcelike { const jsonType = - this._tsFlowOptions.rawType === "json" ? "string" : "any"; + this._tsFlowOptions.rawType === "json" ? "string" : this.anyType(); return [ "function to", name, @@ -212,7 +218,7 @@ export abstract class TypeScriptFlowBaseRenderer extends JavaScriptRenderer { protected serializerFunctionLine(t: Type, name: Name): Sourcelike { const camelCaseName = modifySource(camelCase, name); const returnType = - this._tsFlowOptions.rawType === "json" ? "string" : "any"; + this._tsFlowOptions.rawType === "json" ? "string" : this.anyType(); return [ "function ", camelCaseName, @@ -227,6 +233,10 @@ export abstract class TypeScriptFlowBaseRenderer extends JavaScriptRenderer { return undefined; } + // The runtime typecheck helpers are deliberately dynamic, so they + // stay on `any` even with `prefer-unknown`. Only the public API + // surface (type declarations and converter signatures) uses + // `anyType()`. protected get castFunctionLines(): [string, string] { return [ "function cast(val: any, typ: any): T", diff --git a/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptRenderer.ts b/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptRenderer.ts index 376036727d..1d0d488424 100644 --- a/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptRenderer.ts +++ b/packages/quicktype-core/src/language/TypeScriptFlow/TypeScriptRenderer.ts @@ -9,13 +9,31 @@ import { TypeScriptFlowBaseRenderer } from "./TypeScriptFlowBaseRenderer.js"; import { tsFlowTypeAnnotations } from "./utils.js"; export class TypeScriptRenderer extends TypeScriptFlowBaseRenderer { + protected anyType(): string { + return this._tsFlowOptions.preferUnknown ? "unknown" : "any"; + } + protected forbiddenNamesForGlobalNamespace(): string[] { return ["Array", "Date"]; } + protected uncheckedParsedJson(t: Type, parsedJson: Sourcelike): Sourcelike { + // With `raw-type any` and `prefer-unknown` the deserializer's + // parameter is `unknown`, which can't be returned as the target + // type without a cast. + if ( + this._tsFlowOptions.rawType !== "json" && + this._tsFlowOptions.preferUnknown + ) { + return [parsedJson, " as ", this.sourceFor(t).source]; + } + + return parsedJson; + } + protected deserializerFunctionLine(t: Type, name: Name): Sourcelike { const jsonType = - this._tsFlowOptions.rawType === "json" ? "string" : "any"; + this._tsFlowOptions.rawType === "json" ? "string" : this.anyType(); return [ "public static to", name, @@ -29,7 +47,7 @@ export class TypeScriptRenderer extends TypeScriptFlowBaseRenderer { protected serializerFunctionLine(t: Type, name: Name): Sourcelike { const camelCaseName = modifySource(camelCase, name); const returnType = - this._tsFlowOptions.rawType === "json" ? "string" : "any"; + this._tsFlowOptions.rawType === "json" ? "string" : this.anyType(); return [ "public static ", camelCaseName, diff --git a/packages/quicktype-core/src/language/TypeScriptFlow/language.ts b/packages/quicktype-core/src/language/TypeScriptFlow/language.ts index d7f0f208ba..a894ee2833 100644 --- a/packages/quicktype-core/src/language/TypeScriptFlow/language.ts +++ b/packages/quicktype-core/src/language/TypeScriptFlow/language.ts @@ -45,6 +45,11 @@ export const tsFlowOptions = { false, ), readonly: new BooleanOption("readonly", "Use readonly type members", false), + preferUnknown: new BooleanOption( + "prefer-unknown", + "Use unknown (TypeScript) or mixed (Flow) instead of any", + true, + ), }; export const typeScriptLanguageConfig = { diff --git a/test/fixtures.ts b/test/fixtures.ts index a49aef802f..dece181ef6 100644 --- a/test/fixtures.ts +++ b/test/fixtures.ts @@ -5,6 +5,8 @@ import { randomBytes } from "node:crypto"; import * as shell from "shelljs"; const Ajv = require("ajv"); +const addFormats = require("ajv-formats"); +const draft06MetaSchema = require("ajv/dist/refs/json-schema-draft-06.json"); import { compareJsonFileToJson, @@ -600,17 +602,26 @@ class JSONSchemaJSONFixture extends JSONToXToYFixture { fs.readFileSync(this.language.output, "utf8"), ); - const ajv = new Ajv({ - format: "full", - unknownFormats: ["integer", "boolean"], - }); + const ajv = new Ajv(); + // We generate draft-06 schemas, which Ajv 8 doesn't support out of + // the box anymore. + ajv.addMetaSchema(draft06MetaSchema); + // Ajv 8 moved the format validators into the ajv-formats package; + // its default mode is "full", like the old `format: "full"` option. + addFormats(ajv); + // Our custom schema keywords, which strict mode would reject. + ajv.addVocabulary(["qt-uri-protocols", "qt-uri-extensions"]); // Make Ajv's date-time compatible with what we recognize. All non-standard // JSON formats that we use for transformed type kinds must be registered here - // with a validation function. + // with a validation function. Formats registered with `true` are + // accepted without validating the string. This replaces the old + // `unknownFormats: ["integer", "boolean"]` option. // FIXME: Unify this with what's in StringTypes.ts. ajv.addFormat("date-time", (s: string) => dateTimeRecognizer.isDateTime(s), ); + ajv.addFormat("integer", true); + ajv.addFormat("boolean", true); const valid = ajv.validate(schema, input); if (!valid) { failWith("Generated schema does not validate input JSON.", { @@ -1566,6 +1577,7 @@ export const allFixtures: Fixture[] = [ new JSONFixture(languages.JavaScriptLanguage), new JSONFixture(languages.KotlinLanguage), new JSONFixture(languages.Scala3Language), + new JSONFixture(languages.Scala3UpickleLanguage, "scala3-upickle"), new JSONFixture(languages.KotlinJacksonLanguage, "kotlin-jackson"), new JSONFixture(languages.KotlinXLanguage, "kotlinx"), new JSONFixture(languages.DartLanguage), @@ -1609,6 +1621,10 @@ export const allFixtures: Fixture[] = [ ), new JSONSchemaFixture(languages.KotlinXLanguage, "schema-kotlinx"), new JSONSchemaFixture(languages.Scala3Language), + new JSONSchemaFixture( + languages.Scala3UpickleLanguage, + "schema-scala3-upickle", + ), new JSONSchemaFixture(languages.DartLanguage), new JSONSchemaFixture(languages.PikeLanguage), new JSONSchemaFixture(languages.HaskellLanguage), diff --git a/test/fixtures/csharp-SystemTextJson/test.csproj b/test/fixtures/csharp-SystemTextJson/test.csproj index 1f5224ffd3..37d7e4c935 100755 --- a/test/fixtures/csharp-SystemTextJson/test.csproj +++ b/test/fixtures/csharp-SystemTextJson/test.csproj @@ -1,9 +1,9 @@  Exe - net6 + net8.0 - + diff --git a/test/fixtures/csharp/test.csproj b/test/fixtures/csharp/test.csproj index 2fbf1477c5..9b3c4614b1 100755 --- a/test/fixtures/csharp/test.csproj +++ b/test/fixtures/csharp/test.csproj @@ -1,7 +1,7 @@  Exe - netcoreapp6.0 + net8.0 diff --git a/test/fixtures/python/run.sh b/test/fixtures/python/run.sh index 0ad6e67872..011cad464c 100755 --- a/test/fixtures/python/run.sh +++ b/test/fixtures/python/run.sh @@ -3,7 +3,7 @@ if [ "x$QUICKTYPE_PYTHON_VERSION" = "x2.7" ] ; then PYTHON="python2.7" else - PYTHON="python3.7" + PYTHON="python3.12" fi "$PYTHON" "$@" diff --git a/test/fixtures/scala3-upickle/.gitignore b/test/fixtures/scala3-upickle/.gitignore new file mode 100644 index 0000000000..f252b419dc --- /dev/null +++ b/test/fixtures/scala3-upickle/.gitignore @@ -0,0 +1,3 @@ +main.jar +.bsp +.scala-build diff --git a/test/fixtures/scala3-upickle/run.sh b/test/fixtures/scala3-upickle/run.sh new file mode 100755 index 0000000000..7707d38414 --- /dev/null +++ b/test/fixtures/scala3-upickle/run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +scala-cli upickle.scala TopLevel.scala diff --git a/test/fixtures/scala3-upickle/upickle.scala b/test/fixtures/scala3-upickle/upickle.scala new file mode 100644 index 0000000000..d3e72865a5 --- /dev/null +++ b/test/fixtures/scala3-upickle/upickle.scala @@ -0,0 +1,13 @@ +//> using scala "3.2.2" +//> using dep "com.lihaoyi::upickle:3.1.0" +//> using options "-Xmax-inlines", "500000" + +package quicktype + +@main def main = { + val json = scala.io.Source.fromFile("sample.json").getLines.mkString + val parsed = OptionPickler.read[TopLevel](json) + val jsonString = OptionPickler.writeJs(parsed) + val arr: Array[Byte] = jsonString.toString.getBytes("UTF-8") + System.out.write(arr, 0, arr.length) +} diff --git a/test/fixtures/scala3/circe.scala b/test/fixtures/scala3/circe.scala index b415f95451..a8e4b49867 100644 --- a/test/fixtures/scala3/circe.scala +++ b/test/fixtures/scala3/circe.scala @@ -1,30 +1,23 @@ -//> using scala "3.2.1" -//> using lib "io.circe::circe-core:0.15.0-M1" -//> using lib "io.circe::circe-parser:0.15.0-M1" +//> using scala "3.2.2" +//> using dep "io.circe::circe-core:0.14.5" +//> using dep "io.circe::circe-parser:0.14.5" //> using options "-Xmax-inlines", "3000" + package quicktype import io.circe._ import io.circe.parser._ import io.circe.syntax._ -/* - -case class TopLevel ( - val data : Long, - val next : Option[TopLevel] = None -) derives Encoder.AsObject, Decoder */ - -@main def main = - val json = scala.io.Source.fromFile("sample.json").getLines.mkString - - parse(json).map(x => - val arg = x.as[TopLevel] - arg match { - case Right(y) => - val jsonString = y.asJson - val arr : Array[Byte] = jsonString.toString.getBytes("UTF-8") - System.out.write(arr, 0, arr.length) - case Left(y) => println(y); println("-----");println(y.getMessage) - } - ) +@main def main = { + val json = scala.io.Source.fromFile("sample.json").getLines.mkString + // `scala.` in case the generated code has types named Right/Left. + parse(json).flatMap(_.as[TopLevel]) match { + case scala.Right(y) => + val arr: Array[Byte] = y.asJson.toString.getBytes("UTF-8") + System.out.write(arr, 0, arr.length) + case scala.Left(err) => + System.err.println(err) + sys.exit(1) + } +} diff --git a/test/languages.ts b/test/languages.ts index b64b21cfca..3b4362efb2 100644 --- a/test/languages.ts +++ b/test/languages.ts @@ -118,6 +118,7 @@ export const CSharpLanguage: Language = { quickTestRendererOptions: [ { "array-type": "list" }, { "csharp-version": "5" }, + { "csharp-version": "8" }, { density: "dense" }, { "number-type": "decimal" }, { "any-type": "dynamic" }, @@ -160,14 +161,12 @@ export const CSharpLanguageSystemTextJson: Language = { "minmaxlength.schema", // generated converter triggers CS8602 warnings, which "dotnet run" prints to stdout, breaking the JSON comparison "optional-constraints.schema", // same CS8602 stdout issue; also min/max on integers and pattern on optional strings aren't checked, so expected-failure samples don't fail "optional-const-ref.schema", // same CS8602 stdout issue; also min/max on integers isn't checked, so the expected-failure sample doesn't fail - "required.schema", // the renderer doesn't implement check-required, so the expected-failure sample doesn't fail - "strict-optional.schema", // the renderer doesn't implement check-required, so the expected-failure sample doesn't fail - "intersection.schema", // the renderer doesn't implement check-required, so the expected-failure sample doesn't fail ], rendererOptions: { "check-required": "true", framework: "SystemTextJson" }, quickTestRendererOptions: [ { "array-type": "list" }, { "csharp-version": "6" }, + { "csharp-version": "8" }, { density: "dense" }, { "number-type": "decimal" }, { "any-type": "dynamic" }, @@ -263,7 +262,13 @@ export const PythonLanguage: Language = { "keyword-unions.schema", // Requires more than 255 arguments ], rendererOptions: {}, - quickTestRendererOptions: [{ "python-version": "3.5" }], + quickTestRendererOptions: [ + // The default is "3.10"; keep the older feature sets covered. + { "python-version": "3.5" }, + { "python-version": "3.6" }, + { "python-version": "3.7" }, + { "python-version": "3.9" }, + ], sourceFiles: ["src/language/Python/index.ts"], }; @@ -954,6 +959,7 @@ export const TypeScriptLanguage: Language = { { "acronym-style": "pascal" }, { converters: "all-objects" }, { readonly: "true" }, + { "prefer-unknown": "false" }, ], sourceFiles: ["src/language/TypeScript/index.ts"], }; @@ -1033,6 +1039,7 @@ export const FlowLanguage: Language = { { "runtime-typecheck": "false" }, { "runtime-typecheck-ignore-unknown-properties": "true" }, { "nice-property-names": "true" }, + { "prefer-unknown": "false" }, ], sourceFiles: ["src/language/Flow/index.ts"], }; @@ -1044,57 +1051,81 @@ export const Scala3Language: Language = { return `cp "${sample}" sample.json && ./run.sh`; }, diffViaSchema: true, - skipDiffViaSchema: ["bug427.json", "keywords.json"], + skipDiffViaSchema: [ + "bug427.json", + // These round-trip fine; the code generated via JSON Schema + // orders one property differently (a pre-existing + // alphabetization quirk around renamed keyword properties). + "github-events.json", + "0a91a.json", + "34702.json", + "76ae1.json", + "af2d1.json", + ], allowMissingNull: true, features: ["enum", "union", "no-defaults"], output: "TopLevel.scala", topLevel: "TopLevel", skipJSON: [ - // These tests have "_" as a param name. Scala can't do this? + // The renderer emits raw JSON property names as (backticked) + // Scala identifiers, so empty names, a bare "_", and names + // containing backticks or line separators cannot compile, and + // properties named "None"/"Option" generate case classes that + // shadow the Scala prelude. "blns-object.json", "identifiers.json", "simple-identifiers.json", "keywords.json", + "nst-test-suite.json", - // these actually work as far as I can tell, but seem to fail because properties are sorted differently - // I don't think they fail... but I can't figure out sorting so hey ho let's skip them + // Scala3 has the same prelude-shadowing bug that this input + // guards against in Rust (issue #2521): a field named "options" + // generates `case class Option`, which shadows scala.Option. + "bug2521.json", + ], + skipSchema: [ + // Same raw-identifier limitation as in skipJSON: a property + // named "_" and classes shadowing None/Option don't compile. + "keyword-unions.schema", + ], + skipMiscJSON: false, + rendererOptions: { framework: "circe" }, + quickTestRendererOptions: [], + sourceFiles: ["src/language/Scala3/index.ts"], +}; + +export const Scala3UpickleLanguage: Language = { + name: "scala3", + base: "test/fixtures/scala3-upickle", + runCommand(sample: string) { + return `cp "${sample}" sample.json && ./run.sh`; + }, + diffViaSchema: true, + skipDiffViaSchema: [ + "bug427.json", + // These round-trip fine; the code generated via JSON Schema + // orders one property differently (a pre-existing + // alphabetization quirk around renamed keyword properties). "github-events.json", - "0a358.json", "0a91a.json", "34702.json", "76ae1.json", "af2d1.json", - "bug427.json", - "3d04a0.json", - - // Top level primitives... trivial, - // but annoying as it breaks compilation of the "Top Level" construct... which doesn't exist. - // It's too much hassle to fix - // and has no practical application in this context. Skip. - "no-classes.json", - - // spaces in variables names doesn't seem to work - "name-style.json", - - /* -I havea no idea how to encode these tests correctly. -*/ - "kitchen-sink.json", - "26c9c.json", - "421d4.json", - "a0496.json", - "fcca3.json", - "ae9ca.json", - "617e8.json", - "5f7fe.json", - "f74d5.json", - "a3d8c.json", - "combinations1.json", - "combinations2.json", - "combinations3.json", - "combinations4.json", - "unions.json", - "php-mixed-union.json", + ], + allowMissingNull: true, + features: ["enum", "union", "no-defaults"], + output: "TopLevel.scala", + topLevel: "TopLevel", + skipJSON: [ + // The renderer emits raw JSON property names as (backticked) + // Scala identifiers, so empty names, a bare "_", and names + // containing backticks or line separators cannot compile, and + // properties named "None"/"Option" generate case classes that + // shadow the Scala prelude. + "blns-object.json", + "identifiers.json", + "simple-identifiers.json", + "keywords.json", "nst-test-suite.json", // Scala3 has the same prelude-shadowing bug that this input @@ -1103,25 +1134,12 @@ I havea no idea how to encode these tests correctly. "bug2521.json", ], skipSchema: [ - // 12 skips - "required.schema", - "multi-type-enum.schema", // I think it doesn't correctly realise this is an array of enums. - "integer-string.schema", - "intersection.schema", - ...skipsUntypedUnions, - // The test driver prints the circe DecodingFailure and exits 0, so - // expected-failure samples cannot be detected. - "nested-intersection-union.schema", - "prefix-items.schema", - "date-time-or-string.schema", - "implicit-one-of.schema", - "go-schema-pattern-properties.schema", - ...skipsEnumValueValidation, - "class-with-additional.schema", + // Same raw-identifier limitation as in skipJSON: a property + // named "_" and classes shadowing None/Option don't compile. "keyword-unions.schema", ], skipMiscJSON: false, - rendererOptions: { framework: "circe" }, + rendererOptions: { framework: "upickle" }, quickTestRendererOptions: [], sourceFiles: ["src/language/Scala3/index.ts"], };