From 431c5f466523acb1bf2b188cf441bb6f36804b02 Mon Sep 17 00:00:00 2001 From: "Justin Dane D. Vallar" Date: Mon, 24 Jan 2022 16:56:14 +0800 Subject: [PATCH 1/4] Implement failing tests for floats with trailing point --- test/behavesLikeSqlFormatter.js | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/behavesLikeSqlFormatter.js b/test/behavesLikeSqlFormatter.js index 38cd57156d..8e6bdd7bfc 100644 --- a/test/behavesLikeSqlFormatter.js +++ b/test/behavesLikeSqlFormatter.js @@ -121,7 +121,7 @@ export default function behavesLikeSqlFormatter(format) { expect(result).toBe(dedent` LIMIT 5; - + SELECT foo, bar; @@ -378,7 +378,7 @@ export default function behavesLikeSqlFormatter(format) { Column1 FROM Table1; - + SELECT COUNT(*), Column1 @@ -408,7 +408,7 @@ export default function behavesLikeSqlFormatter(format) { * FROM test; - + CREATE TABLE test( id NUMBER NOT NULL, @@ -429,6 +429,22 @@ export default function behavesLikeSqlFormatter(format) { `); }); + it('correctly handles floats with trailing point', () => { + let result = format('SELECT 1000. AS a;'); + expect(result).toBe(dedent` + SELECT + 1000. AS a; + `); + + result = format('SELECT a, b / 1000. AS a_s, 100. * b / SUM(a_s);'); + expect(result).toBe(dedent` + SELECT + a, + b / 1000. AS a_s, + 100.* b / SUM(a_s); + `); + }); + it('does not split UNION ALL in half', () => { const result = format(` SELECT * FROM tbl1 From c0c1dcb2093583003d1c492bf5751da1a6ec2680 Mon Sep 17 00:00:00 2001 From: "Justin Dane D. Vallar" Date: Mon, 24 Jan 2022 17:14:26 +0800 Subject: [PATCH 2/4] Update tests --- test/behavesLikeSqlFormatter.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/behavesLikeSqlFormatter.js b/test/behavesLikeSqlFormatter.js index 8e6bdd7bfc..a1af7b20f5 100644 --- a/test/behavesLikeSqlFormatter.js +++ b/test/behavesLikeSqlFormatter.js @@ -439,10 +439,10 @@ export default function behavesLikeSqlFormatter(format) { result = format('SELECT a, b / 1000. AS a_s, 100. * b / SUM(a_s);'); expect(result).toBe(dedent` SELECT - a, - b / 1000. AS a_s, - 100.* b / SUM(a_s); - `); + a, + b / 1000. AS a_s, + 100. * b / SUM(a_s); + `); }); it('does not split UNION ALL in half', () => { From 71ab1723003164fe7c0ebc9ed4bc1005f5812230 Mon Sep 17 00:00:00 2001 From: "Justin Dane D. Vallar" Date: Mon, 24 Jan 2022 17:15:51 +0800 Subject: [PATCH 3/4] Fix parsing bug with trailing floating-point --- src/core/Tokenizer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/Tokenizer.ts b/src/core/Tokenizer.ts index 729899ff41..973cafda9a 100644 --- a/src/core/Tokenizer.ts +++ b/src/core/Tokenizer.ts @@ -83,7 +83,7 @@ export default class Tokenizer { [TokenType.LINE_COMMENT]: regexFactory.createLineCommentRegex(cfg.lineCommentTypes), [TokenType.BLOCK_COMMENT]: /^(\/\*[^]*?(?:\*\/|$))/u, [TokenType.NUMBER]: - /^((-\s*)?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u, + /^((-\s*)?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)/u, [TokenType.PLACEHOLDER]: NULL_REGEX, // matches nothing }; From b7636683132dc5f2de0a8528a428bbce3bae9c8f Mon Sep 17 00:00:00 2001 From: Martin Malek Date: Tue, 15 Mar 2022 14:54:12 +0100 Subject: [PATCH 4/4] Return token when pair not closed When something is not closed the tokenizer will try to find next token but will fail. Better to return what was parsed then to fail with error. --- src/core/Tokenizer.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/Tokenizer.ts b/src/core/Tokenizer.ts index 973cafda9a..ef55bf15ff 100644 --- a/src/core/Tokenizer.ts +++ b/src/core/Tokenizer.ts @@ -124,6 +124,10 @@ export default class Tokenizer { if (input.length) { // Get the next token and the token type token = this.getNextToken(input, token); + // Token can be empty + if (!token) { + return tokens; + } // Advance the string input = input.substring(token.value.length);