🧵 lexer: Fix empty string lit in middle of line (#118)

Fix empty string lit in middle of line. As we advance at the beginning
of reading string and then again when peekRune is `"` we overshoot for
empty strings `""`. Fix this be doing the outer advance only at the end
of the iteration.

Pull-Request: #118

Author: juliaogris

pkg/lexer/lexer.go

@@ -193,7 +193,6 @@ func (l *Lexer) readString() (string, error) {
 	pos := l.pos
 	backslashCnt := 0
 	for {
-		l.advance()
 		if l.cur == '\\' {
 			backslashCnt++
 		} else {
@@ -207,6 +206,7 @@ func (l *Lexer) readString() (string, error) {
 		if pr == 0 || pr == '\n' {
 			break // error case
 		}
+		l.advance()
 	}
 	s := string(l.input[pos : l.pos+1])
 	r, err := strconv.Unquote(s)

pkg/lexer/lexer_test.go

@@ -353,3 +353,37 @@ func TestString(t *testing.T) {
 		})
 	}
 }
+
+func TestStringLit(t *testing.T) {
+	in := `fn "" 0`
+	wantTokens := []*Token{
+		{Type: IDENT, Literal: "fn", Offset: 0, Line: 1, Col: 1},
+		{Type: WS, Literal: "", Offset: 2, Line: 1, Col: 3},
+		{Type: STRING_LIT, Literal: "", Offset: 3, Line: 1, Col: 4},
+		{Type: WS, Literal: "", Offset: 5, Line: 1, Col: 6},
+		{Type: NUM_LIT, Literal: "0", Offset: 6, Line: 1, Col: 7},
+	}
+	l := New(in)
+	for _, want := range wantTokens {
+		got := l.Next()
+		assert.Equal(t, want, got)
+	}
+}
+
+func TestStringLitErr(t *testing.T) {
+	in := `fn "unterminated
+"hello"`
+	wantTokens := []*Token{
+		{Type: IDENT, Literal: "fn", Offset: 0, Line: 1, Col: 1},
+		{Type: WS, Literal: "", Offset: 2, Line: 1, Col: 3},
+		{Type: ILLEGAL, Literal: "invalid string", Offset: 3, Line: 1, Col: 4},
+		{Type: NL, Literal: "", Offset: 16, Line: 1, Col: 17},
+		{Type: STRING_LIT, Literal: "hello", Offset: 17, Line: 2, Col: 1},
+		{Type: EOF, Offset: 24, Line: 2, Col: 8},
+	}
+	l := New(in)
+	for _, want := range wantTokens {
+		got := l.Next()
+		assert.Equal(t, want, got)
+	}
+}

pkg/parser/parser_test.go

@@ -1190,6 +1190,18 @@ func TestCalledBuiltinFuncs(t *testing.T) {
 	assert.Equal(t, want, got)
 }
 
+func TestEmptyStringLitArg(t *testing.T) {
+	input := `
+fn "" 0
+
+func fn s:string n:num
+    print s n
+end`
+	parser := newParser(input, testBuiltins())
+	parser.Parse()
+	assertNoParseError(t, parser, input)
+}
+
 func TestDemo(t *testing.T) {
 	input := `
 move 10 10