diff --git a/packages/quicktype-core/src/TargetLanguage.ts b/packages/quicktype-core/src/TargetLanguage.ts index 78b536b583..f0617b7390 100644 --- a/packages/quicktype-core/src/TargetLanguage.ts +++ b/packages/quicktype-core/src/TargetLanguage.ts @@ -15,6 +15,7 @@ import type { Type } from "./Type/Type.js"; import type { StringTypeMapping } from "./Type/TypeBuilderUtils.js"; import type { TypeGraph } from "./Type/TypeGraph.js"; import type { Comment } from "./support/Comments.js"; +import { INT64_RANGE, type IntegerRange } from "./support/IntegerRange.js"; import { defined } from "./support/Support.js"; import type { LanguageName, RendererOptions } from "./types.js"; @@ -121,4 +122,22 @@ export abstract class TargetLanguage< public get dateTimeRecognizer(): DateTimeRecognizer { return new DefaultDateTimeRecognizer(); } + + /** + * The inclusive range of whole numbers in input JSON that quicktype + * infers as the language's integer type. Whole numbers outside the + * range are inferred as `double` instead, because the integer type + * could not round-trip them (issue #2931). `null` means the + * language's integers are arbitrary-precision. + * + * Languages whose integer width depends on a renderer option (like + * cJSON's `integer-size`) override this and inspect + * `rendererOptions`, which are the same untyped option values that + * `makeRenderer` receives. + */ + public getSupportedIntegerRange( + _rendererOptions: Record = {}, + ): IntegerRange | null { + return INT64_RANGE; + } } diff --git a/packages/quicktype-core/src/index.ts b/packages/quicktype-core/src/index.ts index 2b21c7deec..0b0089a13c 100644 --- a/packages/quicktype-core/src/index.ts +++ b/packages/quicktype-core/src/index.ts @@ -16,6 +16,15 @@ export { type InferenceFlagName, } from "./Inference.js"; export { CompressedJSON, type Value } from "./input/CompressedJSON.js"; +export { + INT8_RANGE, + INT16_RANGE, + INT32_RANGE, + INT64_RANGE, + type IntegerRange, + JS_SAFE_INTEGER_RANGE, + integerStringInRange, +} from "./support/IntegerRange.js"; export { type Input, InputData, diff --git a/packages/quicktype-core/src/input/CompressedJSON.ts b/packages/quicktype-core/src/input/CompressedJSON.ts index 09e9193633..e07a6d0d97 100644 --- a/packages/quicktype-core/src/input/CompressedJSON.ts +++ b/packages/quicktype-core/src/input/CompressedJSON.ts @@ -2,6 +2,11 @@ import { addHashCode, hashCodeInit, hashString } from "collection-utils"; import { inferTransformedStringTypeKindForString } from "../attributes/StringTypes.js"; import type { DateTimeRecognizer } from "../DateTime.js"; +import { + INT64_RANGE, + type IntegerRange, + integerStringInRange, +} from "../support/IntegerRange.js"; import { assert, defined, panic } from "../support/Support.js"; import { type TransformedStringTypeKind, @@ -66,13 +71,50 @@ export abstract class CompressedJSON { private readonly _arrays: Value[][] = []; + /** + * `supportedIntegerRange` is the range of whole numbers in the input + * that get inferred as `integer`; whole numbers outside it are inferred + * as `double`, because the target language's integer type could not + * round-trip them. `null` means the target's integers are + * arbitrary-precision. See `TargetLanguage.getSupportedIntegerRange`. + */ public constructor( public readonly dateTimeRecognizer: DateTimeRecognizer, public readonly handleRefs: boolean, + public readonly supportedIntegerRange: IntegerRange | null = INT64_RANGE, ) {} public abstract parse(input: T): Promise; + /** + * Whether a whole number in the input, given as the decimal string of + * its JSON literal, fits `supportedIntegerRange`. Works on the digit + * string because such literals can exceed what a JavaScript number can + * represent exactly. + */ + protected integerStringFits(integerString: string): boolean { + const range = this.supportedIntegerRange; + if (range === null) return true; + return integerStringInRange(integerString, range); + } + + /** + * Whether a number that `JSON.parse` produced should be inferred as + * `double`. The original literal is gone at this point, but for whole + * numbers below 1e21, `toFixed(0)` gives the exact decimal value of the + * double, and it errs on the right side at range boundaries: a literal + * like 9223372036854775807 (INT64_MAX) parses to the double + * 9223372036854775808, which is correctly outside the int64 range. At + * 1e21 doubles are far beyond any fixed-size integer type and `toFixed` + * switches to exponential notation, so those are doubles outright. + */ + protected parsedNumberIsDouble(n: number): boolean { + if (n !== Math.floor(n)) return true; + if (this.supportedIntegerRange === null) return false; + if (Math.abs(n) >= 1e21) return true; + return !this.integerStringFits(n.toFixed(0)); + } + public parseSync(_input: T): Value { return panic("parseSync not implemented in CompressedJSON"); } @@ -340,11 +382,7 @@ export class CompressedJSONFromString extends CompressedJSON { } else if (typeof json === "string") { this.commitString(json); } else if (typeof json === "number") { - const isDouble = - json !== Math.floor(json) || - json < Number.MIN_SAFE_INTEGER || - json > Number.MAX_SAFE_INTEGER; - this.commitNumber(isDouble); + this.commitNumber(this.parsedNumberIsDouble(json)); } else if (Array.isArray(json)) { this.pushArrayContext(); for (const v of json) { diff --git a/packages/quicktype-core/src/input/Inputs.ts b/packages/quicktype-core/src/input/Inputs.ts index 1419fd19b9..65c31448de 100644 --- a/packages/quicktype-core/src/input/Inputs.ts +++ b/packages/quicktype-core/src/input/Inputs.ts @@ -206,6 +206,7 @@ export function jsonInputForTargetLanguage( targetLanguage: LanguageName | TargetLanguage, languages?: TargetLanguage[], handleJSONRefs = false, + rendererOptions: Record = {}, ): JSONInput { if (typeof targetLanguage === "string") { targetLanguage = defined(languageNamed(targetLanguage, languages)); @@ -214,6 +215,7 @@ export function jsonInputForTargetLanguage( const compressedJSON = new CompressedJSONFromString( targetLanguage.dateTimeRecognizer, handleJSONRefs, + targetLanguage.getSupportedIntegerRange(rendererOptions), ); return new JSONInput(compressedJSON); } diff --git a/packages/quicktype-core/src/language/CJSON/language.ts b/packages/quicktype-core/src/language/CJSON/language.ts index 08ef7f893e..6ac2e956a0 100644 --- a/packages/quicktype-core/src/language/CJSON/language.ts +++ b/packages/quicktype-core/src/language/CJSON/language.ts @@ -27,6 +27,13 @@ import { StringOption, getOptionValues, } from "../../RendererOptions/index.js"; +import { + INT8_RANGE, + INT16_RANGE, + INT32_RANGE, + INT64_RANGE, + type IntegerRange, +} from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { LanguageName, RendererOptions } from "../../types.js"; @@ -142,6 +149,28 @@ export class CJSONTargetLanguage extends TargetLanguage< return true; } + /** + * Return the range of whole numbers the generated integer type can + * represent, which depends on the `integer-size` renderer option + * (int64_t by default) + * @param rendererOptions: untyped renderer option values + * @return the range of the configured integer type + */ + public getSupportedIntegerRange( + rendererOptions: Record = {}, + ): IntegerRange | null { + switch (cJSONOptions.typeIntegerSize.getValue(rendererOptions)) { + case "int8_t": + return INT8_RANGE; + case "int16_t": + return INT16_RANGE; + case "int32_t": + return INT32_RANGE; + default: + return INT64_RANGE; + } + } + /** * Indicate if language support optional class properties * @return true diff --git a/packages/quicktype-core/src/language/Crystal/language.ts b/packages/quicktype-core/src/language/Crystal/language.ts index 921ab392c7..6fbb293de6 100644 --- a/packages/quicktype-core/src/language/Crystal/language.ts +++ b/packages/quicktype-core/src/language/Crystal/language.ts @@ -1,4 +1,5 @@ import type { RenderContext } from "../../Renderer.js"; +import { INT32_RANGE, type IntegerRange } from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import { CrystalRenderer } from "./CrystalRenderer.js"; @@ -16,6 +17,11 @@ export class CrystalTargetLanguage extends TargetLanguage< super(crystalLanguageConfig); } + // The Crystal renderer emits `Int32` for inferred integers. + public getSupportedIntegerRange(): IntegerRange | null { + return INT32_RANGE; + } + protected makeRenderer(renderContext: RenderContext): CrystalRenderer { return new CrystalRenderer(this, renderContext); } diff --git a/packages/quicktype-core/src/language/Elixir/language.ts b/packages/quicktype-core/src/language/Elixir/language.ts index 249a6a64d3..065db722a7 100644 --- a/packages/quicktype-core/src/language/Elixir/language.ts +++ b/packages/quicktype-core/src/language/Elixir/language.ts @@ -4,6 +4,7 @@ import { StringOption, getOptionValues, } from "../../RendererOptions/index.js"; +import type { IntegerRange } from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { LanguageName, RendererOptions } from "../../types.js"; @@ -28,6 +29,11 @@ export const elixirLanguageConfig = { export class ElixirTargetLanguage extends TargetLanguage< typeof elixirLanguageConfig > { + // Elixir's integers are arbitrary-precision. + public getSupportedIntegerRange(): IntegerRange | null { + return null; + } + public constructor() { super(elixirLanguageConfig); } diff --git a/packages/quicktype-core/src/language/Elm/language.ts b/packages/quicktype-core/src/language/Elm/language.ts index cac5bad46c..efdbc0601a 100644 --- a/packages/quicktype-core/src/language/Elm/language.ts +++ b/packages/quicktype-core/src/language/Elm/language.ts @@ -5,6 +5,10 @@ import { StringOption, getOptionValues, } from "../../RendererOptions/index.js"; +import { + type IntegerRange, + JS_SAFE_INTEGER_RANGE, +} from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { LanguageName, RendererOptions } from "../../types.js"; @@ -55,6 +59,12 @@ export class ElmTargetLanguage extends TargetLanguage< return true; } + // Elm compiles to JavaScript, where `Int` is an IEEE-754 double at + // runtime, so integers are only exact within the JS safe range. + public getSupportedIntegerRange(): IntegerRange | null { + return JS_SAFE_INTEGER_RANGE; + } + protected makeRenderer( renderContext: RenderContext, untypedOptionValues: RendererOptions, diff --git a/packages/quicktype-core/src/language/JSONSchema/language.ts b/packages/quicktype-core/src/language/JSONSchema/language.ts index 01890ee8cf..f0488ae8fc 100644 --- a/packages/quicktype-core/src/language/JSONSchema/language.ts +++ b/packages/quicktype-core/src/language/JSONSchema/language.ts @@ -1,4 +1,5 @@ import type { RenderContext } from "../../Renderer.js"; +import type { IntegerRange } from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import { type StringTypeMapping, @@ -17,6 +18,11 @@ export const JSONSchemaLanguageConfig = { export class JSONSchemaTargetLanguage extends TargetLanguage< typeof JSONSchemaLanguageConfig > { + // JSON Schema's `integer` type is unbounded. + public getSupportedIntegerRange(): IntegerRange | null { + return null; + } + public constructor() { super(JSONSchemaLanguageConfig); } diff --git a/packages/quicktype-core/src/language/JavaScript/language.ts b/packages/quicktype-core/src/language/JavaScript/language.ts index f8893ac26a..89e7452ebb 100644 --- a/packages/quicktype-core/src/language/JavaScript/language.ts +++ b/packages/quicktype-core/src/language/JavaScript/language.ts @@ -6,6 +6,10 @@ import { } from "../../RendererOptions/index.js"; import { AcronymStyleOptions, acronymOption } from "../../support/Acronyms.js"; import { convertersOption } from "../../support/Converters.js"; +import { + JS_SAFE_INTEGER_RANGE, + type IntegerRange, +} from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { PrimitiveStringTypeKind, @@ -51,6 +55,10 @@ export const javaScriptLanguageConfig = { export class JavaScriptTargetLanguage extends TargetLanguage< typeof javaScriptLanguageConfig > { + public getSupportedIntegerRange(): IntegerRange | null { + return JS_SAFE_INTEGER_RANGE; + } + public constructor() { super(javaScriptLanguageConfig); } diff --git a/packages/quicktype-core/src/language/JavaScriptPropTypes/language.ts b/packages/quicktype-core/src/language/JavaScriptPropTypes/language.ts index 11862b3dbb..179d4c702d 100644 --- a/packages/quicktype-core/src/language/JavaScriptPropTypes/language.ts +++ b/packages/quicktype-core/src/language/JavaScriptPropTypes/language.ts @@ -2,6 +2,10 @@ import type { RenderContext } from "../../Renderer.js"; import { EnumOption, getOptionValues } from "../../RendererOptions/index.js"; import { AcronymStyleOptions, acronymOption } from "../../support/Acronyms.js"; import { convertersOption } from "../../support/Converters.js"; +import { + JS_SAFE_INTEGER_RANGE, + type IntegerRange, +} from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { LanguageName, RendererOptions } from "../../types.js"; @@ -30,6 +34,10 @@ export const javaScriptPropTypesLanguageConfig = { export class JavaScriptPropTypesTargetLanguage extends TargetLanguage< typeof javaScriptPropTypesLanguageConfig > { + public getSupportedIntegerRange(): IntegerRange | null { + return JS_SAFE_INTEGER_RANGE; + } + public constructor() { super(javaScriptPropTypesLanguageConfig); } diff --git a/packages/quicktype-core/src/language/Pike/language.ts b/packages/quicktype-core/src/language/Pike/language.ts index 7ecdd67917..ab3f35d9af 100644 --- a/packages/quicktype-core/src/language/Pike/language.ts +++ b/packages/quicktype-core/src/language/Pike/language.ts @@ -1,4 +1,5 @@ import type { RenderContext } from "../../Renderer.js"; +import type { IntegerRange } from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import { PikeRenderer } from "./PikeRenderer.js"; @@ -14,6 +15,11 @@ export const pikeLanguageConfig = { export class PikeTargetLanguage extends TargetLanguage< typeof pikeLanguageConfig > { + // Pike's integers are arbitrary-precision. + public getSupportedIntegerRange(): IntegerRange | null { + return null; + } + public constructor() { super(pikeLanguageConfig); } diff --git a/packages/quicktype-core/src/language/Python/language.ts b/packages/quicktype-core/src/language/Python/language.ts index 009bbcba0f..101c5115b8 100644 --- a/packages/quicktype-core/src/language/Python/language.ts +++ b/packages/quicktype-core/src/language/Python/language.ts @@ -6,6 +6,7 @@ import { EnumOption, getOptionValues, } from "../../RendererOptions/index.js"; +import type { IntegerRange } from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { StringTypeMapping } from "../../Type/TypeBuilderUtils.js"; import { @@ -57,6 +58,11 @@ export const pythonLanguageConfig = { export class PythonTargetLanguage extends TargetLanguage< typeof pythonLanguageConfig > { + // Python's integers are arbitrary-precision. + public getSupportedIntegerRange(): IntegerRange | null { + return null; + } + public constructor() { super(pythonLanguageConfig); } diff --git a/packages/quicktype-core/src/language/Ruby/language.ts b/packages/quicktype-core/src/language/Ruby/language.ts index f44e714bf4..99b7d1d4ac 100644 --- a/packages/quicktype-core/src/language/Ruby/language.ts +++ b/packages/quicktype-core/src/language/Ruby/language.ts @@ -5,6 +5,7 @@ import { StringOption, getOptionValues, } from "../../RendererOptions/index.js"; +import type { IntegerRange } from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { LanguageName, RendererOptions } from "../../types.js"; @@ -41,6 +42,11 @@ export const rubyLanguageConfig = { export class RubyTargetLanguage extends TargetLanguage< typeof rubyLanguageConfig > { + // Ruby's integers are arbitrary-precision. + public getSupportedIntegerRange(): IntegerRange | null { + return null; + } + public constructor() { super(rubyLanguageConfig); } diff --git a/packages/quicktype-core/src/language/TypeScriptEffectSchema/language.ts b/packages/quicktype-core/src/language/TypeScriptEffectSchema/language.ts index 9d4186722f..2c2d94a842 100644 --- a/packages/quicktype-core/src/language/TypeScriptEffectSchema/language.ts +++ b/packages/quicktype-core/src/language/TypeScriptEffectSchema/language.ts @@ -1,5 +1,9 @@ import type { RenderContext } from "../../Renderer.js"; import { BooleanOption, getOptionValues } from "../../RendererOptions/index.js"; +import { + JS_SAFE_INTEGER_RANGE, + type IntegerRange, +} from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { LanguageName, RendererOptions } from "../../types.js"; @@ -18,6 +22,10 @@ export const typeScriptEffectSchemaLanguageConfig = { export class TypeScriptEffectSchemaTargetLanguage extends TargetLanguage< typeof typeScriptEffectSchemaLanguageConfig > { + public getSupportedIntegerRange(): IntegerRange | null { + return JS_SAFE_INTEGER_RANGE; + } + public constructor() { super(typeScriptEffectSchemaLanguageConfig); } diff --git a/packages/quicktype-core/src/language/TypeScriptFlow/language.ts b/packages/quicktype-core/src/language/TypeScriptFlow/language.ts index 3401829555..d7f0f208ba 100644 --- a/packages/quicktype-core/src/language/TypeScriptFlow/language.ts +++ b/packages/quicktype-core/src/language/TypeScriptFlow/language.ts @@ -1,5 +1,9 @@ import type { RenderContext } from "../../Renderer.js"; import { BooleanOption, getOptionValues } from "../../RendererOptions/index.js"; +import { + JS_SAFE_INTEGER_RANGE, + type IntegerRange, +} from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { StringTypeMapping } from "../../Type/TypeBuilderUtils.js"; import type { @@ -52,6 +56,10 @@ export const typeScriptLanguageConfig = { export class TypeScriptTargetLanguage extends TargetLanguage< typeof typeScriptLanguageConfig > { + public getSupportedIntegerRange(): IntegerRange | null { + return JS_SAFE_INTEGER_RANGE; + } + public constructor() { super(typeScriptLanguageConfig); } @@ -98,6 +106,10 @@ export const flowLanguageConfig = { export class FlowTargetLanguage extends TargetLanguage< typeof flowLanguageConfig > { + public getSupportedIntegerRange(): IntegerRange | null { + return JS_SAFE_INTEGER_RANGE; + } + public constructor() { super(flowLanguageConfig); } diff --git a/packages/quicktype-core/src/language/TypeScriptZod/language.ts b/packages/quicktype-core/src/language/TypeScriptZod/language.ts index 22d5344204..63b9abab51 100644 --- a/packages/quicktype-core/src/language/TypeScriptZod/language.ts +++ b/packages/quicktype-core/src/language/TypeScriptZod/language.ts @@ -1,5 +1,9 @@ import type { RenderContext } from "../../Renderer.js"; import { BooleanOption, getOptionValues } from "../../RendererOptions/index.js"; +import { + JS_SAFE_INTEGER_RANGE, + type IntegerRange, +} from "../../support/IntegerRange.js"; import { TargetLanguage } from "../../TargetLanguage.js"; import type { StringTypeMapping } from "../../Type/TypeBuilderUtils.js"; import type { @@ -23,6 +27,10 @@ export const typeScriptZodLanguageConfig = { export class TypeScriptZodTargetLanguage extends TargetLanguage< typeof typeScriptZodLanguageConfig > { + public getSupportedIntegerRange(): IntegerRange | null { + return JS_SAFE_INTEGER_RANGE; + } + public constructor() { super(typeScriptZodLanguageConfig); } diff --git a/packages/quicktype-core/src/support/IntegerRange.ts b/packages/quicktype-core/src/support/IntegerRange.ts new file mode 100644 index 0000000000..8ac7b770b7 --- /dev/null +++ b/packages/quicktype-core/src/support/IntegerRange.ts @@ -0,0 +1,102 @@ +/** + * An inclusive range of integers that a target language's integer type can + * represent exactly. The bounds are decimal strings because they can lie + * outside the range in which JavaScript numbers are exact — the int64 + * boundaries, for example, are not representable as doubles. + */ +export interface IntegerRange { + readonly max: string; + readonly min: string; +} + +/** + * The range of a signed 64-bit integer, the integer type that most of + * quicktype's target languages use. + */ +export const INT64_RANGE: IntegerRange = { + min: "-9223372036854775808", + max: "9223372036854775807", +}; + +/** + * The ranges of the narrower fixed-size signed integer types, for target + * languages whose integer type is less than 64 bits wide. + */ +export const INT32_RANGE: IntegerRange = { + min: "-2147483648", + max: "2147483647", +}; + +export const INT16_RANGE: IntegerRange = { + min: "-32768", + max: "32767", +}; + +export const INT8_RANGE: IntegerRange = { + min: "-128", + max: "127", +}; + +/** + * The range in which every integer is exactly representable as an IEEE-754 + * double, which is how JavaScript and its relatives represent all numbers. + */ +export const JS_SAFE_INTEGER_RANGE: IntegerRange = { + min: "-9007199254740991", + max: "9007199254740991", +}; + +function splitIntegerString(s: string): { digits: string; negative: boolean } { + const hasSign = s.startsWith("-"); + let digits = hasSign ? s.slice(1) : s; + + let firstNonZero = 0; + while (firstNonZero < digits.length - 1 && digits[firstNonZero] === "0") { + firstNonZero += 1; + } + + digits = digits.slice(firstNonZero); + + // "-0" is just 0. + return { digits, negative: hasSign && digits !== "0" }; +} + +/** + * Compares two integers given as decimal strings, returning a negative + * number, zero, or a positive number as `a` is less than, equal to, or + * greater than `b`. Handles signs and leading zeros, and is exact for + * integers of any size. + */ +export function compareIntegerStrings(a: string, b: string): number { + const sa = splitIntegerString(a); + const sb = splitIntegerString(b); + + if (sa.negative !== sb.negative) { + return sa.negative ? -1 : 1; + } + + const sign = sa.negative ? -1 : 1; + + if (sa.digits.length !== sb.digits.length) { + return sa.digits.length < sb.digits.length ? -sign : sign; + } + + if (sa.digits === sb.digits) { + return 0; + } + + // Equal-length runs of digits compare lexicographically the same way + // they compare numerically. + return sa.digits < sb.digits ? -sign : sign; +} + +/** + * Decides whether an integer, given as its decimal string, lies within + * `range` (inclusive). + */ +export function integerStringInRange(s: string, range: IntegerRange): boolean { + return ( + compareIntegerStrings(s, range.min) >= 0 && + compareIntegerStrings(s, range.max) <= 0 + ); +} diff --git a/src/CompressedJSONFromStream.ts b/src/CompressedJSONFromStream.ts index 892e35920b..67e9c05b48 100644 --- a/src/CompressedJSONFromStream.ts +++ b/src/CompressedJSONFromStream.ts @@ -19,6 +19,11 @@ const methodMap: { [name: string]: string } = { }; export class CompressedJSONFromStream extends CompressedJSON { + // The text of the integer literal being parsed. Numbers cannot nest, + // so a single accumulator suffices. Only consulted when the number is + // not already classified as a double. + private _currentIntegerString = ""; + public async parse(readStream: Readable): Promise { const combo = new Parser({ packKeys: true, packStrings: true }); combo.on( @@ -47,17 +52,27 @@ export class CompressedJSONFromStream extends CompressedJSON { protected handleStartNumber = (): void => { this.pushContext(); this.context.currentNumberIsDouble = false; + this._currentIntegerString = ""; }; protected handleNumberChunk = (s: string): void => { const ctx = this.context; - if (!ctx.currentNumberIsDouble && /[.e]/i.test(s)) { + if (ctx.currentNumberIsDouble) return; + + if (/[.e]/i.test(s)) { ctx.currentNumberIsDouble = true; + } else { + this._currentIntegerString += s; } }; protected handleEndNumber(): void { - const isDouble = this.context.currentNumberIsDouble; + // A whole number outside the target language's integer range must + // be a double — the integer type could not round-trip it + // (https://github.com/glideapps/quicktype/issues/2931). + const isDouble = + this.context.currentNumberIsDouble || + !this.integerStringFits(this._currentIntegerString); this.popContext(); this.commitNumber(isDouble); } diff --git a/src/index.ts b/src/index.ts index 079a257a9e..6ae510e25b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -906,6 +906,7 @@ export function jsonInputForTargetLanguage( targetLanguage: string | TargetLanguage, languages?: TargetLanguage[], handleJSONRefs = false, + rendererOptions: Record = {}, ): JSONInput { if (typeof targetLanguage === "string") { const languageName = isLanguageName(targetLanguage) @@ -917,6 +918,7 @@ export function jsonInputForTargetLanguage( const compressedJSON = new CompressedJSONFromStream( targetLanguage.dateTimeRecognizer, handleJSONRefs, + targetLanguage.getSupportedIntegerRange(rendererOptions), ); return new JSONInput(compressedJSON); } @@ -926,6 +928,7 @@ async function makeInputData( targetLanguage: TargetLanguage, additionalSchemaAddresses: readonly string[], handleJSONRefs: boolean, + rendererOptions: Record, httpHeaders?: string[], ): Promise { const inputData = new InputData(); @@ -945,6 +948,7 @@ async function makeInputData( targetLanguage, undefined, handleJSONRefs, + rendererOptions, ), ); break; @@ -1177,6 +1181,7 @@ export async function makeQuicktypeOptions( lang, options.additionalSchema, quicktypeOptions.ignoreJsonRefs !== true, + options.rendererOptions, options.httpHeader, ); diff --git a/test/unit/integer-range-inference.test.ts b/test/unit/integer-range-inference.test.ts new file mode 100644 index 0000000000..5406ba3163 --- /dev/null +++ b/test/unit/integer-range-inference.test.ts @@ -0,0 +1,263 @@ +// A whole number in input JSON that lies outside the target language's +// integer range must be inferred as `double`, not `integer` — the integer +// type could not round-trip it, so the generated code would fail on the +// very sample it was generated from. The range is language-specific: most +// languages use 64-bit integers, JavaScript and its relatives are only +// exact within ±(2^53 - 1), and languages like Python have +// arbitrary-precision integers with no limit at all. +// See https://github.com/glideapps/quicktype/issues/2931. +// +// The CLI parses JSON with a streaming parser that never materializes the +// number as a JavaScript value, so the range check has to work on the digit +// string; JS numbers cannot represent int64 boundary values exactly. +import { DefaultDateTimeRecognizer } from "quicktype-core/dist/DateTime.js"; +import stringToStream from "string-to-stream"; +import { describe, expect, test } from "vitest"; + +import { + INT16_RANGE, + INT32_RANGE, + INT64_RANGE, + InputData, + type IntegerRange, + JSONInput, + JS_SAFE_INTEGER_RANGE, + type LanguageName, + integerStringInRange, + jsonInputForTargetLanguage, + languageNamed, + quicktype, +} from "quicktype-core"; + +import { CompressedJSONFromStream } from "../../src/CompressedJSONFromStream"; + +describe("integerStringInRange", () => { + test("accepts values inside the int64 range", () => { + expect(integerStringInRange("0", INT64_RANGE)).toBe(true); + expect(integerStringInRange("-0", INT64_RANGE)).toBe(true); + expect(integerStringInRange("1", INT64_RANGE)).toBe(true); + expect(integerStringInRange("-1", INT64_RANGE)).toBe(true); + expect(integerStringInRange("123456789", INT64_RANGE)).toBe(true); + // INT64_MAX and INT64_MIN are still integers. + expect(integerStringInRange("9223372036854775807", INT64_RANGE)).toBe( + true, + ); + expect(integerStringInRange("-9223372036854775808", INT64_RANGE)).toBe( + true, + ); + }); + + test("ignores leading zeros", () => { + expect( + integerStringInRange("00000000000000000000042", INT64_RANGE), + ).toBe(true); + expect( + integerStringInRange("-00000000000000000000042", INT64_RANGE), + ).toBe(true); + expect(integerStringInRange("009223372036854775807", INT64_RANGE)).toBe( + true, + ); + expect(integerStringInRange("009223372036854775808", INT64_RANGE)).toBe( + false, + ); + }); + + test("rejects values just outside the int64 range", () => { + expect(integerStringInRange("9223372036854775808", INT64_RANGE)).toBe( + false, + ); + expect(integerStringInRange("-9223372036854775809", INT64_RANGE)).toBe( + false, + ); + }); + + test("rejects values far outside the int64 range", () => { + expect( + integerStringInRange("123456789012345678901234567890", INT64_RANGE), + ).toBe(false); + expect( + integerStringInRange( + "-123456789012345678901234567890", + INT64_RANGE, + ), + ).toBe(false); + }); + + test("the JS safe-integer range is narrower than int64", () => { + expect( + integerStringInRange("9007199254740991", JS_SAFE_INTEGER_RANGE), + ).toBe(true); + expect( + integerStringInRange("9007199254740992", JS_SAFE_INTEGER_RANGE), + ).toBe(false); + expect( + integerStringInRange("-9007199254740991", JS_SAFE_INTEGER_RANGE), + ).toBe(true); + expect( + integerStringInRange("-9007199254740992", JS_SAFE_INTEGER_RANGE), + ).toBe(false); + }); +}); + +// Mirror how the CLI wires up JSON input (jsonInputForTargetLanguage in +// src/index.ts): the streaming CompressedJSON parser is where integer +// vs. double classification happens, using the target language's range. +async function streamedLinesForJSON( + lang: LanguageName, + json: string, + range: IntegerRange | null, +): Promise { + const compressedJSON = new CompressedJSONFromStream( + new DefaultDateTimeRecognizer(), + false, + range, + ); + const input = new JSONInput(compressedJSON); + await input.addSource({ name: "Edge", samples: [stringToStream(json)] }); + const inputData = new InputData(); + inputData.addInput(input); + const result = await quicktype({ inputData, lang }); + return result.lines.join("\n"); +} + +function rangeForLanguage(lang: LanguageName): IntegerRange | null { + const language = languageNamed(lang); + if (language === undefined) { + throw new Error(`no such language: ${lang}`); + } + + return language.getSupportedIntegerRange(); +} + +describe("language-declared integer ranges", () => { + test("Crystal renders integers as Int32, so its range is int32", () => { + expect(rangeForLanguage("crystal")).toEqual(INT32_RANGE); + }); + + test("Elm's Int is a JavaScript number at runtime", () => { + expect(rangeForLanguage("elm")).toEqual(JS_SAFE_INTEGER_RANGE); + }); + + test("cJSON's range follows the integer-size renderer option", () => { + const cjson = languageNamed("cjson"); + if (cjson === undefined) throw new Error("no such language: cjson"); + expect(cjson.getSupportedIntegerRange()).toEqual(INT64_RANGE); + expect( + cjson.getSupportedIntegerRange({ "integer-size": "int16_t" }), + ).toEqual(INT16_RANGE); + }); +}); + +describe("streaming inference of numbers at the integer-range boundary", () => { + test("Go: out-of-range whole numbers become float64, INT64_MAX stays int64", async () => { + const lines = await streamedLinesForJSON( + "go", + '{"big": 9223372036854775807, "bigger": 123456789012345678901234567890}', + rangeForLanguage("go"), + ); + expect(lines).toMatch(/Big\s+\*?int64/); + expect(lines).toMatch(/Bigger\s+\*?float64/); + }); + + test("Go: INT64_MIN stays int64, one below becomes float64", async () => { + const lines = await streamedLinesForJSON( + "go", + '{"min": -9223372036854775808, "smaller": -9223372036854775809}', + rangeForLanguage("go"), + ); + expect(lines).toMatch(/Min\s+\*?int64/); + expect(lines).toMatch(/Smaller\s+\*?float64/); + }); + + test("Crystal: whole numbers beyond Int32 become Float64", async () => { + const lines = await streamedLinesForJSON( + "crystal", + '{"fits": 2147483647, "overflows": 2147483648}', + rangeForLanguage("crystal"), + ); + expect(lines).toMatch(/fits.*Int32/); + expect(lines).toMatch(/overflows.*Float64/); + }); + + test("Python: arbitrary-precision integers never overflow to float", async () => { + const lines = await streamedLinesForJSON( + "python", + '{"huge": 123456789012345678901234567890}', + rangeForLanguage("python"), + ); + expect(lines).toMatch(/huge:\s+int/); + expect(lines).not.toMatch(/huge:\s+float/); + }); + + // The same literal classifies differently under different ranges; the + // Python renderer distinguishes int from float, which makes the + // classification visible in the output. + test("the range is what decides, not the language's renderer", async () => { + const literal = '{"n": 9007199254740993}'; // 2^53 + 1: in int64, outside JS-safe + + const asInt64 = await streamedLinesForJSON( + "python", + literal, + INT64_RANGE, + ); + expect(asInt64).toMatch(/n:\s+int/); + + const asJsSafe = await streamedLinesForJSON( + "python", + literal, + JS_SAFE_INTEGER_RANGE, + ); + expect(asJsSafe).toMatch(/n:\s+float/); + }); +}); + +describe("core (JSON.parse-based) inference at the integer-range boundary", () => { + async function coreLinesForJSON( + lang: LanguageName, + json: string, + rendererOptions: Record = {}, + ): Promise { + const jsonInput = jsonInputForTargetLanguage( + lang, + undefined, + false, + rendererOptions, + ); + await jsonInput.addSource({ name: "Edge", samples: [json] }); + const inputData = new InputData(); + inputData.addInput(jsonInput); + const result = await quicktype({ inputData, lang, rendererOptions }); + return result.lines.join("\n"); + } + + test("Go: whole numbers between 2^53 and INT64_MAX stay int64", async () => { + // 2^63 - 1024, the largest double below INT64_MAX. + const lines = await coreLinesForJSON( + "go", + '{"big": 9223372036854774784, "bigger": 123456789012345678901234567890}', + ); + expect(lines).toMatch(/Big\s+\*?int64/); + expect(lines).toMatch(/Bigger\s+\*?float64/); + }); + + test("Python: huge whole numbers stay int", async () => { + const lines = await coreLinesForJSON( + "python", + '{"huge": 123456789012345678901234567890}', + ); + expect(lines).toMatch(/huge:\s+int/); + }); + + test("cJSON: the integer-size option narrows the inferred range", async () => { + const json = '{"n": 32768}'; // fits int64_t, not int16_t + + const asDefault = await coreLinesForJSON("cjson", json); + expect(asDefault).toMatch(/int64_t\s+\*?n/); + + const asInt16 = await coreLinesForJSON("cjson", json, { + "integer-size": "int16_t", + }); + expect(asInt16).toMatch(/double\s+\*?n/); + expect(asInt16).not.toMatch(/int16_t\s+\*?n/); + }); +});