diff --git a/src/core/Tokenizer.ts b/src/core/Tokenizer.ts index 729899ff41..ef55bf15ff 100644 --- a/src/core/Tokenizer.ts +++ b/src/core/Tokenizer.ts @@ -83,7 +83,7 @@ export default class Tokenizer { [TokenType.LINE_COMMENT]: regexFactory.createLineCommentRegex(cfg.lineCommentTypes), [TokenType.BLOCK_COMMENT]: /^(\/\*[^]*?(?:\*\/|$))/u, [TokenType.NUMBER]: - /^((-\s*)?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u, + /^((-\s*)?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)/u, [TokenType.PLACEHOLDER]: NULL_REGEX, // matches nothing }; @@ -124,6 +124,10 @@ export default class Tokenizer { if (input.length) { // Get the next token and the token type token = this.getNextToken(input, token); + // Token can be empty + if (!token) { + return tokens; + } // Advance the string input = input.substring(token.value.length); diff --git a/test/behavesLikeSqlFormatter.js b/test/behavesLikeSqlFormatter.js index 38cd57156d..a1af7b20f5 100644 --- a/test/behavesLikeSqlFormatter.js +++ b/test/behavesLikeSqlFormatter.js @@ -121,7 +121,7 @@ export default function behavesLikeSqlFormatter(format) { expect(result).toBe(dedent` LIMIT 5; - + SELECT foo, bar; @@ -378,7 +378,7 @@ export default function behavesLikeSqlFormatter(format) { Column1 FROM Table1; - + SELECT COUNT(*), Column1 @@ -408,7 +408,7 @@ export default function behavesLikeSqlFormatter(format) { * FROM test; - + CREATE TABLE test( id NUMBER NOT NULL, @@ -429,6 +429,22 @@ export default function behavesLikeSqlFormatter(format) { `); }); + it('correctly handles floats with trailing point', () => { + let result = format('SELECT 1000. AS a;'); + expect(result).toBe(dedent` + SELECT + 1000. AS a; + `); + + result = format('SELECT a, b / 1000. AS a_s, 100. * b / SUM(a_s);'); + expect(result).toBe(dedent` + SELECT + a, + b / 1000. AS a_s, + 100. * b / SUM(a_s); + `); + }); + it('does not split UNION ALL in half', () => { const result = format(` SELECT * FROM tbl1