Return token when pair not closed

src/core/Tokenizer.ts

@@ -83,7 +83,7 @@ export default class Tokenizer {
 			[TokenType.LINE_COMMENT]: regexFactory.createLineCommentRegex(cfg.lineCommentTypes),
 			[TokenType.BLOCK_COMMENT]: /^(\/\*[^]*?(?:\*\/|$))/u,
 			[TokenType.NUMBER]:
-				/^((-\s*)?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u,
+				/^((-\s*)?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)/u,
 			[TokenType.PLACEHOLDER]: NULL_REGEX, // matches nothing
 		};
 
@@ -124,6 +124,10 @@ export default class Tokenizer {
 			if (input.length) {
 				// Get the next token and the token type
 				token = this.getNextToken(input, token);
+				// Token can be empty
+				if (!token) {
+					return tokens;
+				}
 				// Advance the string
 				input = input.substring(token.value.length);
 

test/behavesLikeSqlFormatter.js

@@ -121,7 +121,7 @@ export default function behavesLikeSqlFormatter(format) {
 		expect(result).toBe(dedent`
       LIMIT
         5;
-      
+
       SELECT
         foo,
         bar;
@@ -378,7 +378,7 @@ export default function behavesLikeSqlFormatter(format) {
         Column1
       FROM
         Table1;
-      
+
       SELECT
         COUNT(*),
         Column1
@@ -408,7 +408,7 @@ export default function behavesLikeSqlFormatter(format) {
         *
       FROM
         test;
-      
+
       CREATE TABLE
         test(
           id NUMBER NOT NULL,
@@ -429,6 +429,22 @@ export default function behavesLikeSqlFormatter(format) {
     `);
 	});
 
+	it('correctly handles floats with trailing point', () => {
+		let result = format('SELECT 1000. AS a;');
+		expect(result).toBe(dedent`
+		  SELECT
+		    1000. AS a;
+		`);
+
+		result = format('SELECT a, b / 1000. AS a_s, 100. * b / SUM(a_s);');
+		expect(result).toBe(dedent`
+		  SELECT
+		    a,
+		    b / 1000. AS a_s,
+		    100. * b / SUM(a_s);
+	  `);
+	});
+
 	it('does not split UNION ALL in half', () => {
 		const result = format(`
       SELECT * FROM tbl1